OSDev.org

Posted: **Sun Jul 13, 2025 5:36 pm**

I'm trying to experiment using VMX to load a proprietary gpu driver from linux to my OS, now I just want to start simple it returns 7 on vmlaunch
the code is running in kernel mode, uefi, 64 Bit. VMWARE.

Code: Select all

void VmStart(void* VmxMem) {
    _disable();
    EnhancedMemClr(VmxMem, 0x8000);
    UINT32* VmxOn = VmxMem;
    UINT32* VmCs = (UINT32*)((char*)VmxMem + 0x1000);
    UINT32 RevId = (UINT32)(__readmsr(0x480) & 0x7FFFFFFF);
    BOOTDATA->SimpleConOut(L" VM BASIC %lx FEATURE_CONTROL %lx VMXONRegion %lx", __readmsr(0x480), __readmsr(0x3A), VmxOn);
    BOOTDATA->SimpleConOut(L"VMEXITHANDLER %lx GDTR B %lx L %lx IDTR B %lx L %lx  TSS %lx", VmExitHandler, __GDTR.base, (UINT64)__GDTR.limit, __IDTR.base, (UINT64)__IDTR.limit, &__TSS);
    *VmxOn = RevId;
    *VmCs = RevId;
    int Code;
    if((Code = _VmxOn(VmxOn)))
    {   
        BOOTDATA->SimpleConOut(L"VMXON Failed %d", Code);
    }
        UINT64 error;

    __vmx_vmread(0x4400 /* VM_INSTRUCTION_ERROR */, &error);
BOOTDATA->SimpleConOut(L"VMERR = %llu", error);

    UINT64 GuestStack = (UINT64)VmxMem + 0x5000;

    __vmx_vmwrite(GUEST_CR0, __readcr0());
    __vmx_vmwrite(GUEST_CR4, __readcr4());
    __vmx_vmwrite(GUEST_CR3, __readcr3());
    __vmx_vmwrite(GUEST_CS_SELECTOR, 0x8);
    __vmx_vmwrite(GUEST_RIP, (UINT64)_VmxGuest);
    __vmx_vmwrite(GUEST_RSP, GuestStack);
    __vmx_vmwrite(GUEST_RFLAGS, __readeflags());

    // Host state
    __vmx_vmwrite(HOST_CR0, __readcr0());
    __vmx_vmwrite(HOST_CR3, __readcr3());
    __vmx_vmwrite(HOST_CR4, __readcr4());
    __vmx_vmwrite(HOST_CS_SELECTOR, 0x8);
    __vmx_vmwrite(HOST_SS_SELECTOR, 0x10);
    __vmx_vmwrite(HOST_DS_SELECTOR, 0x10);
    __vmx_vmwrite(HOST_RSP, (UINT64)GuestStack + 0x2000);
    __vmx_vmwrite(HOST_RIP, (UINT64)VmExitHandler);
gvmcs(HOST_IA32_SYSENTER_CS, 0x10);  // kernel data segment
gvmcs(HOST_IA32_SYSENTER_EIP, (UINT64)VmExitHandler);  // or 0
gvmcs(HOST_IA32_SYSENTER_ESP, (UINT64)GuestStack +0x2000);      // valid stack
    if((Code = _VmxVmLaunch())) {
__vmx_vmread(0x4400 /* VM_INSTRUCTION_ERROR */, &error);
BOOTDATA->SimpleConOut(L"VMLAUNCH Err = %llu", error);
    }
}

Posted: **Mon Jul 14, 2025 1:59 am**

Have you executed the VMCLEAR instruction before VMLAUNCH?

Posted: **Mon Jul 14, 2025 4:47 am**

This is vmxon function yes I do that: (use msvc abi) first argument in RCX as far as I know
Vmxon->vmclear->vmptrld->vmread(error) returns 0 ->vmcs setup -> vmlaunch (returns 2 with vmread error returning 7) Even when trying it on real hardware does the same thing (vmx is enabled properly as it seems)

Code: Select all

SECTION .text

global _VmxGuest
global _VmxOn

_VmxOn:
    push rcx
    vmxon [rsp]
    jbe .fail
    add QWORD [rsp], 0x1000
    vmclear [rsp]
    jbe .fail
    vmptrld [rsp]
    jbe .fail


    pop rcx
    xor rax, rax
    ret
.fail:
    pop rcx
    mov rax, 1
    ret

global _VmxVmLaunch

_VmxVmLaunch:
    vmlaunch
    jbe .fail
    xor rax, rax
    ret
.fail:
    mov rax, 1
    ret



[BITS 16]
align 0x1000
_VmxGuest:
    hlt

Vmx enabling code :

Code: Select all

 int regs[4];
    __cpuid(regs, 1);
    BOOLEAN Supported = (regs[2] & (1 << 5)) != 0; // ECX bit 5
    if(!Supported) return 2; // VMX Not supported
    UINT64 Msr = __readmsr(0x3A);
    if((Msr & 1)) {
        if(!(Msr & (1 << 2))) return 3; // VMX Disabled
    } else {
        __writemsr(0x3A, Msr | (5)); // Manually Enable VMX
    }
    
    __writecr0((__readcr0() & __readmsr(IA32_VMX_CR0_FIXED1)) | __readmsr(IA32_VMX_CR0_FIXED0));
    __writecr4(__readcr4() | (1ULL << 13)); // VMXE)
    __writecr4((__readcr4() & __readmsr(IA32_VMX_CR4_FIXED1)) | __readmsr(IA32_VMX_CR4_FIXED0));
    
    return 0; // VMX Is usable

Posted: **Mon Jul 14, 2025 8:57 am**

Update I made it work by disabling EPT in Secondary VM Exec Control.
Now when I vmlaunch it immediately vmexit (on vmware) with code 0x80000021 (if u mask it u get 0x21=HLT), even if I do not do hlt, and none of the code I wrote in vmentry executes even though I read RIP And Instructions at RIP and they all are correct. I'm trying to run directly in long mode.

Code: Select all

void USERAPI VmStart(void* vmxon_region)
{
    void* vmcs_region = (char*)vmxon_region + 0x1000;
    void* guest_stack = (char*)vmxon_region + 0x2000;
    // 1. Read VMX basic info and VMX controls from MSRs
    UINT64 vmx_basic = __readmsr(IA32_VMX_BASIC);
    UINT64 pinbased_ctls = __readmsr(IA32_VMX_TRUE_PINBASED_CTLS);
    UINT64 procbased_ctls = __readmsr(IA32_VMX_TRUE_PROCBASED_CTLS);
    UINT64 secondary_ctls = __readmsr(IA32_VMX_SECONDARY_PROCBASED_CTLS);
    UINT64 vmexit_ctls = __readmsr(IA32_VMX_TRUE_EXIT_CTLS);
    UINT64 vmentry_ctls = __readmsr(IA32_VMX_TRUE_ENTRY_CTLS);

    // Helper: must set fixed 1 bits and clear fixed 0 bits from MSR (upper 32 bits = fixed1 mask, lower 32 bits = fixed0 mask)
    #define FIX_CTL(val, msr) \
        (((val) | (UINT32)((msr) >> 32)) & (UINT32)(msr))

    UINT32 pinbased = FIX_CTL(0, pinbased_ctls);       // start with 0 or with some bits you want enabled
    UINT32 procbased = FIX_CTL(0, procbased_ctls);
    UINT32 secondary = FIX_CTL(0, secondary_ctls);
    UINT32 vmexit = FIX_CTL(0, vmexit_ctls);
    UINT32 vmentry = FIX_CTL(0, vmentry_ctls);

    // // Enable secondary controls bit in primary processor-based controls (bit 31)
    // procbased |= (1u << 31);

    // Enable IA-32e mode guest (bit 9) in VM-entry controls (for 64-bit guest)
    vmentry |= (1u << 9);

    // Enable VM-exit controls for host address space size (bit 9)
    vmexit |= (1u << 9);

    // Optionally enable EPT (bit 1) in secondary controls if you want to use it
    // secondary |= (1u << 1);

    // 2. Prepare VMXON region (must write VMCS revision ID to first 4 bytes)
    *(UINT32*)vmxon_region = (UINT32)(vmx_basic & 0xFFFFFFFF);

    // 3. Prepare VMCS region similarly
    *(UINT32*)vmcs_region = (UINT32)(vmx_basic & 0xFFFFFFFF);

    // 4. Execute VMXON
    int vmxon_status = __vmx_on((UINT64*)&vmxon_region);
    if (vmxon_status != 0) {
        // error
        return;
    }

    // 5. Clear VMCS region before use
    __vmx_vmclear((UINT64*)&vmcs_region);

    // 6. Load VMCS pointer
    __vmx_vmptrld((UINT64*)&vmcs_region);

    // 7. Write VMX control fields to VMCS
    __vmx_vmwrite(PIN_BASED_VM_EXEC_CONTROL, pinbased);
    __vmx_vmwrite(CPU_BASED_VM_EXEC_CONTROL, procbased);
    __vmx_vmwrite(SECONDARY_VM_EXEC_CONTROL, secondary);
    __vmx_vmwrite(VM_EXIT_CONTROLS, vmexit);
    __vmx_vmwrite(VM_ENTRY_CONTROLS, vmentry);

    // 8. Setup guest state fields
    __vmx_vmwrite(GUEST_CR0, __readcr0());
    __vmx_vmwrite(GUEST_CR3, __readcr3());
    __vmx_vmwrite(GUEST_CR4, __readcr4());

    vmcs(GUEST_GDTR_BASE, HOST_GDTR_BASE, __GDTR.base);
    vmcs(GUEST_GDTR_LIMIT, HOST_IA32_SYSENTER_ESP,  __GDTR.limit);
    
    vmcs(GUEST_IDTR_BASE, HOST_IDTR_BASE, __IDTR.base);
    vmcs(GUEST_IDTR_LIMIT, HOST_IA32_SYSENTER_EIP,  __IDTR.limit);

    __vmx_vmwrite(GUEST_LDTR_SELECTOR, 0);
__vmx_vmwrite(GUEST_LDTR_BASE, 0);
__vmx_vmwrite(GUEST_LDTR_LIMIT, 0);
__vmx_vmwrite(GUEST_LDTR_AR_BYTES, 0);

    __vmx_vmwrite(GUEST_CS_SELECTOR, 0x8);
    __vmx_vmwrite(GUEST_DS_SELECTOR, 0x10);
    __vmx_vmwrite(GUEST_ES_SELECTOR, 0x10);
    __vmx_vmwrite(GUEST_GS_SELECTOR, 0x10);
    __vmx_vmwrite(GUEST_FS_SELECTOR, 0x10);
    __vmx_vmwrite(GUEST_SS_SELECTOR, 0x10);

    __vmx_vmwrite(GUEST_RIP, (UINT64)_VmxGuest);
    __vmx_vmwrite(GUEST_RSP, (UINT64)guest_stack);
    __vmx_vmwrite(GUEST_RFLAGS, __readeflags());
    __vmx_vmwrite(GUEST_IA32_EFER, __readmsr(0xC0000080));

    // TODO: similarly setup other guest segments (DS, ES, SS, FS, GS, TR, LDTR)

    // 9. Setup host state fields
    __vmx_vmwrite(HOST_CR0, __readcr0());
    __vmx_vmwrite(HOST_CR3, __readcr3());
    __vmx_vmwrite(HOST_CR4, __readcr4());
    __vmx_vmwrite(HOST_CS_SELECTOR, 0x10); // kernel data segment selector
    __vmx_vmwrite(HOST_SS_SELECTOR, 0x10);
    __vmx_vmwrite(HOST_DS_SELECTOR, 0x10);
    __vmx_vmwrite(HOST_ES_SELECTOR, 0x10);
    __vmx_vmwrite(HOST_FS_SELECTOR, 0x10);
    __vmx_vmwrite(HOST_GS_SELECTOR, 0x10);
    __vmx_vmwrite(HOST_TR_SELECTOR, 0x18); // TSS selector
    __vmx_vmwrite(HOST_IA32_EFER, __readmsr(0xC0000080));  // MSR_EFER
    __vmx_vmwrite(HOST_RSP, (UINT64)guest_stack + 0x1008);
    __vmx_vmwrite(HOST_RIP, (UINT64)VmExitHandler);

    UINT64 GuestRip;
    __vmx_vmread(GUEST_RIP, &GuestRip);
    BOOTDATA->SimpleConOut(L"Launching VM... GUEST_RIP %lx", GuestRip);
    // 10. Launch the VM
    int vmlaunch_status = __vmx_vmlaunch();
    if (vmlaunch_status != 0) {
        UINT64 vmerr;
        __vmx_vmread(0x4400, &vmerr); // VM-instruction error field
        BOOTDATA->SimpleConOut(L"VMLAUNCH Error %d", vmerr);
        // Handle error - print vmerr, halt, etc.
    }

    // VM launched successfully; guest will run now
}

It returns VM Exit immedialy, instruction length 0 exit code (0x21=HLT) Probably a triple fault because this is the vmguest function :

Code: Select all


align 0x1000
_VmxGuest:
    jmp $
    mov rax, 0xFFFFFFFFFFFFFFFFF
    jmp _VmxGuest

    hlt
    hlt

Posted: **Mon Jul 14, 2025 2:37 pm**

did set the whole vmcs region to 0x00 before setting up the structure? depending on the allocation method it might still contain values stored from previous allocations, which could then lead to undefined guest behavior.

when it exits, are you able to vmresume again?
furthermore can you log the GUEST_RIP on VMExit.

Posted: **Wed Jul 16, 2025 10:15 am**

vm exit code 0x80000021:
0x21 = 33 = VM-entry failure due to invalid guest state
0x80000000 = bit 31. = VM-entry failure (0 = true VM exit; 1 = VM-entry failure)
could you pls dump all guest state values so I can lookup what's wrong?
you are running in long mode, aren't you? EFER.LMA=EFER.LME=CR4.PAE = 1 (both guest as well host)
then you also need to enable 1 bit in vm entry controls telling it is 64 bit mode (it looks redundant but it is required) and the same telling vm exit will be into 64 bit mode
VM-Entry Controls bit 9. IA-32e mode guest
VM-Exit Controls bit 9. 9 Host address-space size
if you are running compatibility mode guest (real mode, 32 bit protected mode and so on) then the vm entry bit 9. should be 0 (guest EFER.LMA=0 or guest EFER.LME=0 or guest CR4.PAE = 0) and moreover running real mode guest requires also enabled EPT and enabled unrestricted guest mode (the guest is running with a strange settings CR0.PG = 1 and CR0.PE=0 which is possible only under virtualization)
I see you enabled bits 9. for vm entry controls as well vm exit controls, that's OK when both guest and host are running in 64 bit

Posted: **Wed Jul 16, 2025 8:01 pm**

I almost abandonned that for later, here is the latest code I had

Now it does not return 7 rather it vmexit with code 0x(80…)21 qualification 0 and if I set vmxe qualification = 4 immediately after vmlaunch before any instructions, I read instructions on vmexit the instructions seem correct in memory.

Code: Select all


#pragma pack(push, 1)
typedef struct {
    UINT16 limit;
    UINT64 base;
} DescriptorTableRegister;
#pragma pack(pop)
// TR (TSS descriptor)
typedef struct {
    UINT16 lim, addr0;
    UINT8 AddrFlagsEtc[4];
    UINT32 Addr2;
    UINT32 Zero;
} TSSDESC;
extern UINT64 __GDT;
int KCALL _VmxOn(void* VmxMem);


// writes to both guest and host
#define SEG_DATA_ACCESS 0xC093  // Present, Read/Write, DPL=0, usable
#define SEG_CODE_ACCESS 0xA09B  // Present, Exec/Read, DPL=0, 64-bit
#define SEG_LIMIT 0xFFFFFFFF
extern DescriptorTableRegister __GDTR, __IDTR;
extern UINT64 __TSS, __TSSR;
#define IA32_VMX_CR0_FIXED0 0x486
#define IA32_VMX_CR0_FIXED1 0x487
#define IA32_VMX_CR4_FIXED0 0x488
#define IA32_VMX_CR4_FIXED1 0x489


#define vmxwrchk(_gr, _val) {static int __errcode; if((__errcode = __vmx_vmwrite(_gr, _val))) {KConOut(L"VMX Write to %lx Failed with code %d", _gr, __errcode);}}

#define vmcs(_guestreg, _hostreg, val) {vmxwrchk((UINT64)_guestreg, (UINT64)(val)); vmxwrchk((UINT64)(_hostreg), (UINT64)(val));}
#define gvmcs(__guestreg, __val) vmxwrchk((UINT64)(__guestreg), (UINT64)(__val))

int K

Code: Select all


void KCALL VmStart(void* vmxon_region)
{
    EnhancedMemClr(vmxon_region, 0x10000);
    void* vmcs_region = (char*)vmxon_region + 0x1000;
    void* guest_stack = (char*)vmxon_region + 0x2000;
    // 1. Read VMX basic info and VMX controls from MSRs
    UINT64 vmx_basic = __readmsr(IA32_VMX_BASIC);
    UINT64 pinbased_ctls = __readmsr(IA32_VMX_TRUE_PINBASED_CTLS);
    UINT64 procbased_ctls = __readmsr(IA32_VMX_TRUE_PROCBASED_CTLS);
    UINT64 secondary_ctls = __readmsr(IA32_VMX_SECONDARY_PROCBASED_CTLS);
    UINT64 vmexit_ctls = __readmsr(IA32_VMX_TRUE_EXIT_CTLS);
    UINT64 vmentry_ctls = __readmsr(IA32_VMX_TRUE_ENTRY_CTLS);

    // Helper: must set fixed 1 bits and clear fixed 0 bits from MSR (upper 32 bits = fixed1 mask, lower 32 bits = fixed0 mask)
    #define FIX_CTL(val, msr) \
        (((val) | (UINT32)((msr) >> 32)) & (UINT32)(msr))

    UINT32 pinbased = FIX_CTL(0, pinbased_ctls);       // start with 0 or with some bits you want enabled
    UINT32 procbased = FIX_CTL(0, procbased_ctls);
    UINT32 secondary = FIX_CTL(0, secondary_ctls);
    UINT32 vmexit = FIX_CTL(0, vmexit_ctls);
    UINT32 vmentry = FIX_CTL(0, vmentry_ctls);

    // secondary controls bit in primary processor-based controls (bit 31)
    procbased |= (1u << 31);

    // Enable IA-32e mode guest (bit 9) in VM-entry controls (for 64-bit guest)
    vmentry |= (1u << 9) | (1u << 14);

    // Enable VM-exit controls for host address space size (bit 9)
    vmexit |= (1u << 9);

    // Optionally enable EPT (bit 1) in secondary controls if you want to use it
    // secondary |= (1u << 1);
secondary |= (1 << 7);
    // 2. Prepare VMXON region (must write VMCS revision ID to first 4 bytes)
    *(UINT32*)vmxon_region = (UINT32)(vmx_basic & 0xFFFFFFFF);

    // 3. Prepare VMCS region similarly
    *(UINT32*)vmcs_region = (UINT32)(vmx_basic & 0xFFFFFFFF);

    // 4. Execute VMXON
    int vmxon_status = __vmx_on((UINT64*)&vmxon_region);
    if (vmxon_status != 0) {
        // error
        return;
    }

    KConOut(L"KERNEL PHYS %lx %lx", GetPhysicalAddress((void*)__GDTR.base), *((UINT64*)GetPhysicalAddress((void*)__GDTR.base) + 8));

    // 5. Clear VMCS region before use
    __vmx_vmclear((UINT64*)&vmcs_region);

    // 6. Load VMCS pointer
    __vmx_vmptrld((UINT64*)&vmcs_region);

    // 7. Write VMX control fields to VMCS
    gvmcs(PIN_BASED_VM_EXEC_CONTROL, pinbased);
    gvmcs(CPU_BASED_VM_EXEC_CONTROL,  procbased);
    gvmcs(SECONDARY_VM_EXEC_CONTROL, secondary);
    gvmcs(VM_EXIT_CONTROLS, vmexit);
    gvmcs(VM_ENTRY_CONTROLS, (1ULL << 9) | (1ULL << 14) | vmentry);
    // 8. Setup guest state fields

    UINT64 guest_cr0 = (__readmsr(IA32_VMX_CR0_FIXED0)) & __readmsr(IA32_VMX_CR0_FIXED1);
UINT64 guest_cr4 = (__readmsr(IA32_VMX_CR4_FIXED0)) & __readmsr(IA32_VMX_CR4_FIXED1);
    gvmcs(GUEST_CR0, (guest_cr0 | (1 << 0)) & ~(1ULL << 31));
    gvmcs(GUEST_CR4, guest_cr4);
    // gvmcs(GUEST_CR3, __readcr3());
    __vmx_vmwrite(GUEST_DR7, 0x00000400);        // DR7: default, no breakpoints
__vmx_vmwrite(0x681E, 0xFFFF0FF0);        // DR6: reserved bits set
__vmx_vmwrite(GUEST_IA32_DEBUGCTL, 0x0);     // Disable debug features

    gvmcs(GUEST_GDTR_BASE, GetPhysicalAddress((void*)__GDTR.base));
    gvmcs(GUEST_GDTR_LIMIT, __GDTR.limit);
    
    gvmcs(GUEST_IDTR_BASE, GetPhysicalAddress((void*)__IDTR.base));
    gvmcs(GUEST_IDTR_LIMIT, __IDTR.limit);
    
    gvmcs(GUEST_LDTR_SELECTOR, 0);
gvmcs(GUEST_LDTR_BASE, 0);
gvmcs(GUEST_LDTR_LIMIT, 0);
// gvmcs(GUEST_LDTR_AR_BYTES, 0x10000);
gvmcs(GUEST_LDTR_AR_BYTES, 0x0000);


    gvmcs(GUEST_CS_SELECTOR, 0x8);
    gvmcs(GUEST_DS_SELECTOR, 0x10);
    gvmcs(GUEST_ES_SELECTOR, 0x10);
    gvmcs(GUEST_GS_SELECTOR, 0x10);
    gvmcs(GUEST_FS_SELECTOR, 0x10);

    gvmcs(GUEST_RIP, (UINT64)GetPhysicalAddress((void*)_VmxGuest));
    gvmcs(GUEST_RSP, (UINT64)GetPhysicalAddress((void*)guest_stack));
    gvmcs(GUEST_RFLAGS, 2);
    gvmcs(GUEST_IA32_EFER, 0x500);

    // TODO: similarly setup other guest segments (DS, ES, SS, FS, GS, TR, LDTR)

    #define SEG_ACCESS_CODE64  0xA09B  // exec/read, conforming=0, long=1
#define SEG_ACCESS_DATA    0xC093  // read/write, expand down=0
#define SEG_ACCESS_TSS     0x008B  // 64-bit TSS (Available)

// CS
gvmcs(GUEST_CS_BASE, 0);
gvmcs(GUEST_CS_LIMIT, 0xFFFFF );
gvmcs(GUEST_CS_AR_BYTES, SEG_ACCESS_CODE64);

// SS, DS, ES, FS, GS
__vmx_vmwrite(GUEST_SS_SELECTOR, 0x10);
__vmx_vmwrite(GUEST_SS_LIMIT, (UINT64)0xFFFFF );
__vmx_vmwrite(GUEST_SS_BASE, 0);
__vmx_vmwrite(GUEST_SS_AR_BYTES, 0xC093);

gvmcs(GUEST_DS_BASE, 0);
gvmcs(GUEST_DS_LIMIT, 0xFFFFF );
gvmcs(GUEST_DS_AR_BYTES, SEG_ACCESS_DATA);

gvmcs(GUEST_ES_BASE, 0);
gvmcs(GUEST_ES_LIMIT, 0xFFFFF );
gvmcs(GUEST_ES_AR_BYTES, SEG_ACCESS_DATA);

gvmcs(GUEST_FS_BASE, 0);
gvmcs(GUEST_FS_LIMIT, 0xFFFFF );
gvmcs(GUEST_FS_AR_BYTES, SEG_ACCESS_DATA);

gvmcs(GUEST_GS_BASE, 0);
gvmcs(GUEST_GS_LIMIT, 0xFFFFF );
gvmcs(GUEST_GS_AR_BYTES, SEG_ACCESS_DATA);

UINT64 TSS = GetPhysicalAddress((void*)(&__TSS));

TSSDESC* Tssr = (TSSDESC*)((char*)(&__GDT) + 0x28);

KConOut(L"LIM %x", (UINT32)Tssr->lim);
Tssr->lim = 0x67;
Tssr->addr0 = (UINT16)TSS;
Tssr->AddrFlagsEtc[0] = (UINT8)(TSS >> 16);
Tssr->AddrFlagsEtc[1] = 0x8B;
Tssr->AddrFlagsEtc[2] = 0b10000000;
Tssr->AddrFlagsEtc[3] = (UINT8)(TSS >> 24);
Tssr->Addr2 = TSS >> 32;
Tssr->Zero = 0;

gvmcs(GUEST_TR_SELECTOR, 0x28);



gvmcs(GUEST_TR_BASE, (UINT64)TSS);  // you must define this
gvmcs(GUEST_TR_LIMIT, 0x67);
gvmcs(GUEST_TR_AR_BYTES, 0x8B);


    // 9. Setup host state fields
    gvmcs(HOST_CR0, __readcr0());
    gvmcs(HOST_CR3, __readcr3());
    gvmcs(HOST_CR4, __readcr4());
    gvmcs(HOST_CS_SELECTOR, 0x10); // kernel data segment selector
    gvmcs(HOST_SS_SELECTOR, 0x10);
    gvmcs(HOST_DS_SELECTOR, 0x10);
    gvmcs(HOST_ES_SELECTOR, 0x10);
    gvmcs(HOST_FS_SELECTOR, 0x10);
    gvmcs(HOST_GS_SELECTOR, 0x10);
    gvmcs(HOST_TR_SELECTOR, 0x18); // TSS selector
    gvmcs(HOST_IA32_EFER, __readmsr(0xC0000080));  // MSR_EFER
    gvmcs(HOST_RSP, (UINT64)guest_stack + 0x1008);
    gvmcs(HOST_RIP, (UINT64)VmExitHandler);

    UINT64 GuestRip;
    __vmx_vmread(GUEST_RIP, &GuestRip);
    KConOut(L"Launching VM... GUEST_RIP %lx", GuestRip);
    // 10. Launch the VM
    int vmlaunch_status = __vmx_vmlaunch();
    if (vmlaunch_status != 0) {
        UINT64 vmerr;
        __vmx_vmread(0x4400, &vmerr); // VM-instruction error field
        KConOut(L"VMLAUNCH Error %d", vmerr);
        // Handle error - print vmerr, halt, etc.
    }

    // VM launched successfully; guest will run now
}

Posted: **Thu Jul 17, 2025 10:15 am**

Code: Select all

// gvmcs(GUEST_LDTR_AR_BYTES, 0x10000);
gvmcs(GUEST_LDTR_AR_BYTES, 0x0000);

it is OK to setup LDTR access rights field with 0x10000 = unusable when the LDTR is not used, why did you disable the first line and used the second one?

these 3 lines at various positions in your code:

Code: Select all

gvmcs(VM_ENTRY_CONTROLS, (1ULL << 9) | (1ULL << 14) | vmentry);
gvmcs(GUEST_CR0, (guest_cr0 | (1 << 0)) & ~(1ULL << 31));
gvmcs(GUEST_IA32_EFER, 0x500);

does the second line mean that you would like to run guest with CR0 bit 0. (PE) enabled and force bit 31. (PG) to be disabled?
the first and third line setup guest into 64 bit mode (AMD calls it long mode and intel calls it IA-32e mode)
on hardware (I mean not running virtualization) if CR0.PG bit is disabled then the EFER cannot hold value 0x500, in such case the value is 0x100 (EFER.LME=1, EFER.LMA=0)
if running under virtualization, this is written in the manual:
If the "IA-32e mode guest" VM-entry control is 1, bit 31 in the CR0 field (corresponding to CR0.PG) and bit 5 in the CR4 field (corresponding to CR4.PAE) must each be 1

In my previous post I asked you to dump all vmcs guest fields in binary to inspect them. I also asked you in which mode you would like to run guest, is it 64 bit mode ?

Posted: **Thu Jul 17, 2025 12:30 pm**

Yeah I forgot I gave u an attempt of implementing unrestricted guest because chatgpt proposed that. But I just want to startup the vm in long mode.

Even if I enable pg and such and set ldtr unused bit it still crashes yes I want to run the vm in long mode directly and maybe load ovmf and linux and get a gpu driver to work from that.

OSDev.org

VMLAUNCH Returns 7

VMLAUNCH Returns 7

Re: VMLAUNCH Returns 7

Re: VMLAUNCH Returns 7

Re: VMLAUNCH Returns 7

Re: VMLAUNCH Returns 7

Re: VMLAUNCH Returns 7

Re: VMLAUNCH Returns 7

Re: VMLAUNCH Returns 7

Re: VMLAUNCH Returns 7