vmm/core/asm.h

/*
 * Copyright (c) 2007, 2008 University of Tsukuba
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 * 3. Neither the name of the University of Tsukuba nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
/*
 * Copyright (c) 2010-2012 Yuichi Watanabe
 */

#ifndef _CORE_ASM_H
#define _CORE_ASM_H

#include <core/linkage.h>
#include "desc.h"
#include <core/types.h>

struct vt_vmentry_regs {
        ulong rax, rcx, rdx, rbx, cr2, rbp, rsi, rdi;
        ulong r8, r9, r10, r11, r12, r13, r14, r15;
        ulong cr3;
        int pe, pg;
        struct {
                int enable, num;
                u16 es, cs, ss, ds, fs, gs;
        } sw;
};

struct svm_vmrun_regs {
        ulong cr0, rcx, rdx, rbx, cr4, rbp, rsi, rdi;
        ulong r8, r9, r10, r11, r12, r13, r14, r15;
        ulong cr3;
};

#define SW_SREG_ES_BIT (1 << 0)
#define SW_SREG_CS_BIT (1 << 1)
#define SW_SREG_SS_BIT (1 << 2)
#define SW_SREG_DS_BIT (1 << 3)
#define SW_SREG_FS_BIT (1 << 4)
#define SW_SREG_GS_BIT (1 << 5)

#define ASM_VMCALL "vmcall"

asmlinkage int asm_vmlaunch_regs_32 (struct vt_vmentry_regs *p);
asmlinkage int asm_vmresume_regs_32 (struct vt_vmentry_regs *p);
asmlinkage int asm_vmlaunch_regs_64 (struct vt_vmentry_regs *p);
asmlinkage int asm_vmresume_regs_64 (struct vt_vmentry_regs *p);
asmlinkage void asm_vmrun_regs_32 (struct svm_vmrun_regs *p, ulong vmcb_phys,
                                   ulong vmcbhost_phys);
asmlinkage void asm_vmrun_regs_64 (struct svm_vmrun_regs *p, ulong vmcb_phys,
                                   ulong vmcbhost_phys);

static inline void
asm_cpuid (u32 num, u32 numc, u32 *a, u32 *b, u32 *c, u32 *d)
{
        asm volatile ("cpuid"
                      : "=a" (*a), "=b" (*b), "=c" (*c), "=d" (*d)
                      : "a" (num), "c" (numc));
}

static inline void
asm_rdmsr32 (ulong num, u32 *a, u32 *d)
{
        asm volatile ("rdmsr"
                      : "=a" (*a), "=d" (*d)
                      : "c" (num));
}

static inline void
asm_wrmsr32 (ulong num, u32 a, u32 d)
{
        asm volatile ("wrmsr"
                      :
                      : "c" (num), "a" (a), "d" (d));
}

static inline void
asm_rdmsr64 (ulong num, u64 *value)
{
        u32 a, d;

        asm_rdmsr32 (num, &a, &d);
        *value = (u64)a | ((u64)d << 32);
}

static inline void
asm_wrmsr64 (ulong num, u64 value)
{
        u32 a, d;

        a = (u32)value;
        d = (u32)(value >> 32);
        asm_wrmsr32 (num, a, d);
}

static inline void
asm_rdmsr (ulong num, ulong *value)
{
#ifdef __x86_64__
        u64 value64;
        asm_rdmsr64 (num, &value64);
        *value = value64;
#else
        u32 value32;
        u32 dummy;
        asm_rdmsr32 (num, &value32, &dummy);
        *value = value32;
#endif
}

static inline void
asm_wrmsr (ulong num, ulong value)
{
#ifdef __x86_64__
        asm_wrmsr64 (num, (u64)value);
#else
        asm_wrmsr32 (num, (u32)value, 0);
#endif
}

static inline void
asm_rdcr0 (ulong *cr0)
{
        asm volatile ("mov %%cr0,%0"
                      : "=r" (*cr0));
}

static inline void
asm_wrcr0 (ulong cr0)
{
        asm volatile ("mov %0,%%cr0"
                      :
                      : "r" (cr0));
}

static inline void
asm_rdcr2 (ulong *cr2)
{
        asm volatile ("mov %%cr2,%0"
                      : "=r" (*cr2));
}

static inline void
asm_wrcr2 (ulong cr2)
{
        asm volatile ("mov %0,%%cr2"
                      :
                      : "r" (cr2));
}

static inline void
asm_rdcr3 (ulong *cr3)
{
        asm volatile ("mov %%cr3,%0"
                      : "=r" (*cr3));
}

static inline void
asm_wrcr3 (ulong cr3)
{
        asm volatile ("mov %0,%%cr3"
                      :
                      : "r" (cr3));
}

static inline void
asm_rdcr4 (ulong *cr4)
{
        asm volatile ("mov %%cr4,%0"
                      : "=r" (*cr4));
}

static inline void
asm_wrcr4 (ulong cr4)
{
        asm volatile ("mov %0,%%cr4"
                      :
                      : "r" (cr4));
}

#ifdef __x86_64__
static inline void
asm_rdcr8 (u64 *cr8)
{
        asm volatile ("mov %%cr8,%0"
                      : "=r" (*cr8));
}

static inline void
asm_wrcr8 (ulong cr8)
{
        asm volatile ("mov %0,%%cr8"
                      :
                      : "r" (cr8));
}
#endif

/* f3 0f c7 33             vmxon  (%ebx) */
static inline void
asm_vmxon (void *vmxon_region)
{
        asm volatile ("vmxon %0"
                      :
                      : "m" (*(ulong *)vmxon_region)
                      : "cc", "memory");
}

/* 0f 01 c4                vmxoff  */
static inline void
asm_vmxoff (void)
{
        asm volatile ("vmxoff"
                      :
                      :
                      : "cc");
}

/* 66 0f c7 33             vmclear (%ebx) */
static inline void
asm_vmclear (void *p)
{
#ifdef AS_DOESNT_SUPPORT_VMX
        asm volatile (".byte 0x66, 0x0f, 0xc7, 0x33"
                      :
                      : "b" (p)
                      : "cc", "memory");
#else
        asm volatile ("vmclear %0"
                      :
                      : "m" (*(ulong *)p)
                      : "cc", "memory");
#endif
}

/* 0f c7 33                vmptrld (%ebx) */
static inline void
asm_vmptrld (void *p)
{
        asm volatile ("vmptrld %0"
                      :
                      : "m" (*(ulong *)p)
                      : "cc", "memory");
}

/* 0f c7 3b                vmptrst (%ebx) */
static inline void
asm_vmptrst (void *p)
{
        asm volatile ("vmptrst %0"
                      :
                      : "m" (*(ulong *)p)
                      : "cc", "memory");
}

/* 0f 79 c2                vmwrite %edx,%eax */
static inline void
asm_vmwrite (ulong index, ulong val)
{
#ifdef AS_DOESNT_SUPPORT_VMX
        asm volatile (".byte 0x0f, 0x79, 0xc2"
                      :
                      : "a" (index), "d" (val)
                      : "cc");
#else
        asm volatile ("vmwrite %1,%0"
                      :
                      : "r" (index), "rm" (val)
                      : "cc");
#endif
}

static inline void
asm_vmwrite32(ulong index, u32 val)
{
        ulong ulong_val = val;
        asm volatile ("vmwrite %1,%0"
                      :
                      : "r" (index), "rm" (ulong_val)
                      : "cc");
}

static inline void
asm_vmwrite64(ulong index, u64 val)
{
#ifdef __x86_64__
        asm volatile ("vmwrite %1,%0"
                      :
                      : "r" (index), "rm" (val)
                      : "cc");
#else
        ulong low, high;
        low = val;
        high = val >> 32;
        asm_vmwrite(index, low);
        asm_vmwrite(index + 1, high);
#endif
}

/* 0f 01 c2                vmlaunch  */
static inline int
asm_vmlaunch_regs (struct vt_vmentry_regs *p)
{
#ifdef __x86_64__
        return asm_vmlaunch_regs_64 (p);
#else
        return asm_vmlaunch_regs_32 (p);
#endif
}

/* 0f 01 c3                vmresume  */
static inline int
asm_vmresume_regs (struct vt_vmentry_regs *p)
{
#ifdef __x86_64__
        return asm_vmresume_regs_64 (p);
#else
        return asm_vmresume_regs_32 (p);
#endif
}

/* 0f 78 c2                vmread %eax,%edx */
static inline void
asm_vmread (ulong index, ulong *val)
{
        asm volatile ("vmread %1,%0"
                      : "=rm" (*val)
                      : "r" (index)
                      : "cc");
}

static inline void
asm_vmread32(ulong index, u32 *val)
{
        ulong ulong_val;
        asm volatile ("vmread %1,%0"
                      : "=rm" (ulong_val)
                      : "r" (index)
                      : "cc");
        *val = ulong_val;
}

static inline void
asm_vmread64(ulong index, u64 *val)
{
#ifdef __x86_64__
        asm volatile ("vmread %1,%0"
                      : "=rm" (*val)
                      : "r" (index)
                      : "cc");
#else
        ulong low, high;
        asm_vmread(index, &low);
        asm_vmread(index + 1, &high);
        *val = low | (high << 32);
#endif
}

static inline void
asm_invept(void *desc)
{
#ifdef __x86_64__
        asm volatile ("invept (%0), %%rax"
                      :
                      : "r" (desc),
                        "a" (1)
                      : "cc");
#else
        asm volatile ("invept (%0), %%eax"
                      :
                      : "r" (desc),
                        "a" (1)
                      : "cc");
#endif
}

static inline void
asm_rdes (u16 *es)
{
        asm volatile ("mov %%es,%0"
                      : "=rm" (*es));
}

static inline void
asm_rdcs (u16 *cs)
{
        asm volatile ("mov %%cs,%0"
                      : "=rm" (*cs));
}

static inline void
asm_rdss (u16 *ss)
{
        asm volatile ("mov %%ss,%0"
                      : "=rm" (*ss));
}

static inline void
asm_rdds (u16 *ds)
{
        asm volatile ("mov %%ds,%0"
                      : "=rm" (*ds));
}

static inline void
asm_rdfs (u16 *fs)
{
        asm volatile ("mov %%fs,%0"
                      : "=rm" (*fs));
}

static inline void
asm_rdgs (u16 *gs)
{
        asm volatile ("mov %%gs,%0"
                      : "=rm" (*gs));
}

static inline void
asm_wres (u16 es)
{
        asm volatile ("mov %0, %%es"
                      :
                      : "rm" ((ulong)es));
}

static inline void
asm_wrcs (u16 cs)
{
#ifdef __x86_64__
        asm volatile ("pushq %0; push $1f; lretq; 1:"
                      :
                      : "g" ((ulong)cs));
#else
        asm volatile ("pushl %0; push $1f; lret; 1:"
                      :
                      : "g" ((ulong)cs));
#endif
}

static inline void
asm_wrss (u16 ss)
{
        asm volatile ("mov %0, %%ss"
                      :
                      : "rm" ((ulong)ss));
}

static inline void
asm_wrds (u16 ds)
{
        asm volatile ("mov %0, %%ds"
                      :
                      : "rm" ((ulong)ds));
}

static inline void
asm_wrfs (u16 fs)
{
        asm volatile ("mov %0, %%fs"
                      :
                      : "rm" ((ulong)fs));
}

static inline void
asm_wrgs (u16 gs)
{
        asm volatile ("mov %0, %%gs"
                      :
                      : "rm" ((ulong)gs));
}

static inline void
asm_rdldtr (u16 *ldtr)
{
        asm volatile ("sldt %0"
                      : "=rm" (*ldtr));
}

static inline void
asm_wrldtr (u16 ldtr)
{
        asm volatile ("lldt %0"
                      :
                      : "rm" (ldtr));
}

static inline void
asm_rdtr (u16 *tr)
{
        asm volatile ("str %0"
                      : "=rm" (*tr));
}

static inline void
asm_wrtr (u16 tr)
{
        asm volatile ("ltr %0"
                      :
                      : "rm" (tr));
}

static inline void
asm_lsl (u32 sel, ulong *limit)
{
        asm volatile ("lsl %1,%0"
                      : "=r" (*limit)
                      : "rm" ((ulong)sel)); /* avoid assembler bug */
}

static inline void
asm_lar (u32 sel, ulong *ar)
{
        asm volatile ("lar %1,%0"
                      : "=r" (*ar)
                      : "rm" ((ulong)sel)); /* avoid assembler bug */
}

static inline void
asm_rdgdtr (ulong *gdtbase, ulong *gdtlimit)
{
        struct descreg gdtr;

        asm volatile ("sgdt %0"
                      : "=m" (gdtr));
        *gdtbase = gdtr.base;
        *gdtlimit = gdtr.limit;
}

static inline void
asm_rdidtr (ulong *idtbase, ulong *idtlimit)
{
        struct descreg idtr;

        asm volatile ("sidt %0"
                      : "=m" (idtr));
        *idtbase = idtr.base;
        *idtlimit = idtr.limit;
}

static inline void
asm_wrgdtr (ulong gdtbase, ulong gdtlimit)
{
        struct descreg gdtr;

        gdtr.base = gdtbase;
        gdtr.limit = gdtlimit;
        asm volatile ("lgdt %0"
                      :
                      : "m" (gdtr));
}

static inline void
asm_wridtr (ulong idtbase, ulong idtlimit)
{
        struct descreg idtr;

        idtr.base = idtbase;
        idtr.limit = idtlimit;
        asm volatile ("lidt %0"
                      :
                      : "m" (idtr));
}

static inline void
asm_rddr7 (ulong *dr7)
{
        asm volatile ("mov %%dr7,%0"
                      : "=r" (*dr7));
}

static inline void
asm_rdrflags (ulong *rflags)
{
        asm volatile (
#ifdef __x86_64__
                "pushfq ; popq %0"
#else
                "pushfl ; popl %0"
#endif
                : "=rm" (*rflags));
}

static inline void
asm_inb (ioport_t port, u8 *data)
{
        asm volatile ("inb %%dx,%%al"
                      : "=a" (*data)
                      : "d" (port));
}

static inline void
asm_inw (ioport_t port, u16 *data)
{
        asm volatile ("inw %%dx,%%ax"
                      : "=a" (*data)
                      : "d" (port));
}

static inline void
asm_inl (ioport_t port, u32 *data)
{
        asm volatile ("inl %%dx,%%eax"
                      : "=a" (*data)
                      : "d" (port));
}

static inline void
asm_outb (ioport_t port, u8 data)
{
        asm volatile ("outb %%al,%%dx"
                      :
                      : "a" (data)
                      , "d" (port));
}

static inline void
asm_outw (ioport_t port, u16 data)
{
        asm volatile ("outw %%ax,%%dx"
                      :
                      : "a" (data)
                      , "d" (port));
}

static inline void
asm_outl (ioport_t port, u32 data)
{
        asm volatile ("outl %%eax,%%dx"
                      :
                      : "a" (data)
                      , "d" (port));
}

static inline void
asm_cli (void)
{
        asm volatile ("cli");
}

static inline void
asm_sti (void)
{
        asm volatile ("sti");
}

static inline void
asm_wrrsp_and_jmp (ulong rsp, void *jmpto)
{
        asm volatile (
#ifdef __x86_64__
                "mov %0,%%rsp;"
                "xor %%rbp,%%rbp;"
                "push %%rbp;"
                "jmp *%1"
#else
                "mov %0,%%esp;"
                "xor %%ebp,%%ebp;"
                "push %%ebp;"
                "jmp *%1"
#endif
                :
                : "g" (rsp)
                , "r" (jmpto));
}

static inline void
asm_wrrsp_and_ret (ulong rsp, ulong rax)
{
        asm volatile (
#ifdef __x86_64__
                "mov %0,%%rsp; ret"
#else
                "mov %0,%%esp; ret"
#endif
                :
                : "g" (rsp)
                , "a" (rax));
}

static inline void
asm_wbinvd (void)
{
        asm volatile ("wbinvd");
}

static inline void
asm_invlpg (void *p)
{
        asm volatile ("invlpg %0"
                      :
                      : "m" (*(u8 *)p));
}

static inline void
asm_cli_and_hlt (void)
{
        asm volatile ("cli; hlt");
}

static inline void
asm_pause (void)
{
        asm volatile ("pause");
}

static inline void
asm_rdrsp (ulong *rsp)
{
#ifdef __x86_64__
        asm volatile ("mov %%rsp,%0" : "=rm" (*rsp));
#else
        asm volatile ("mov %%esp,%0" : "=rm" (*rsp));
#endif
}

static inline void
asm_rdtsc (u32 *a, u32 *d)
{
        asm volatile ("rdtsc" : "=a" (*a), "=d" (*d));
}

static inline void
asm_mul_and_div (u32 mul1, u32 mul2, u32 div1, u32 *quotient, u32 *remainder)
{
        asm volatile ("mull %4 ; divl %5"
                      : "=&a" (*quotient)
                      , "=&d" (*remainder)
                      : "0" (mul1)
                      , "1" (0)
                      , "rm" (mul2)
                      , "rm" (div1)
                      : "cc");
}

static inline void
asm_lock_incl (u32 *d)
{
        asm volatile ("lock incl %0" : "=m" (*d));
}

/*
  if (*dest == *cmp) {
      *dest = eq;
      return false;
  } else {
      *cmp = *dest;
      return true;
  }
*/
static inline bool
asm_lock_cmpxchgl (u32 *dest, u32 *cmp, u32 eq)
{
        int tmp = 0;
 
        asm volatile ("lock cmpxchgl %4,%1 ; je 1f ; inc %2 ; 1:"
                      : "=&a" (*cmp)
                      , "=m" (*dest)
                      , "=&r" (tmp)
                      : "0" (*cmp)
                      , "r" (eq)
                      , "2" (0)
                      : "memory", "cc");
        return (bool)tmp;
}

static inline bool
asm_lock_cmpxchgq (u64 *dest, u64 *cmp, u64 eq)
{
        int tmp;

#ifdef __x86_64__
        asm volatile ("lock cmpxchgq %4,%1 ; je 1f ; inc %2 ; 1:"
                      : "=&a" (*cmp)
                      , "=m" (*dest)
                      , "=&r" (tmp)
                      : "0" (*cmp)
                      , "r" (eq)
                      , "2" (0)
                      : "memory", "cc");
#else
        asm volatile ("lock cmpxchg8b %1 ; je 1f ; inc %2 ; 1:"
                      : "=&A" (*cmp)
                      , "=m" (*dest)
                      , "=&r" (tmp)
                      : "0" (*cmp)
                      , "b" ((u32)eq)
                      , "c" ((u32)(eq >> 32))
                      , "2" (0)
                      : "memory", "cc");
#endif
        return (bool)tmp;
}

/* old = *mem; *mem = newval; return old; */
static inline ulong
asm_lock_swap_ulong (ulong *mem, ulong newval)
{
        ulong oldval;

        asm volatile ("xchg %0,%1"
                      : "=r" (oldval)
                      , "=m" (*mem)
                      : "0" (newval));
        return oldval;
}

static inline ulong
asm_lock_test_ulong (ulong *mem, int bit)
{
        ulong val;
        asm volatile ("bt %2,%0\n"
                      "setc %%al"
                      : "=m" (*mem)
                      , "=r" (val)
                      : "r" (bit)
                      , "1" (0));
        return val;
}

static inline ulong
asm_lock_test_and_set_ulong (ulong *mem, int bit)
{
        ulong oldval;
        asm volatile ("lock bts %2,%0\n"
                      "setc %%al"
                      : "=m" (*mem)
                      , "=a" (oldval)
                      : "r" (bit)
                      , "1" (0));
        return oldval;
}

static inline ulong
asm_lock_test_and_clear_ulong (ulong *mem, int bit)
{
        ulong oldval;
        asm volatile ("lock btr %2,%0\n"
                      "setc %%al"
                      : "=m" (*mem)
                      , "=a" (oldval)
                      : "r" (bit)
                      , "1" (0));
        return oldval;
}

/*
 * Bit scan reverse.
 * Find the most significant set bit.
 */
static inline long
asm_bsr_ulong (ulong *mem)
{
        long bit;
        if (*mem == 0) {
                return -1;
        }
        asm volatile ("bsr %1,%0\n"
                      : "=r" (bit)
                      : "m" (*mem));
        return bit;
}

/*
 * BIT scan forward.
 * Find the least significant set bit.
 */
static inline long
asm_bsf_ulong (ulong *mem)
{
        long bit;
        if (*mem == 0) {
                return -1;
        }
        asm volatile ("bsf %1,%0\n"
                      : "=r" (bit)
                      : "m" (*mem));
        return bit;
}

/* 0f 01 d8                vmrun */
static inline void
asm_vmrun_regs (struct svm_vmrun_regs *p, ulong vmcb_phys, ulong vmcbhost_phys)
{
#ifdef __x86_64__
        asm_vmrun_regs_64 (p, vmcb_phys, vmcbhost_phys);
#else
        asm_vmrun_regs_32 (p, vmcb_phys, vmcbhost_phys);
#endif
}


static inline void
asm_monitor (void *addr, u32 c, u32 d)
{
        asm volatile ("monitor"
                      :: "a" (addr), "c" (c), "d" (d));
}

static inline void
asm_mwait (u32 a, u32 c)
{
        asm volatile ("mwait"
                      ::"a" (a), "c" (c));
}

#endif