Starting additional processors (cores) at low level
Posted: Thu Dec 27, 2018 7:26 am
I try to teach my self a little bit of low level programming. At the moment I try to understand how SMP and starting additional processors work.
I think I understand, that the primary processor sends some APIC interprocessor interrupts to start additional processors (or cores). The additional CPU starts in real mode and has to use some "trampoline" to switch to protected mode.
I think I also understand that GRUB2 runs only on the primary processor. So I decided (again for the sole purpose of teaching myself) to write a grub module that starts additional CPUs, executes some little code on it and stop them again. I have already written user GRUB modules (again, to teach myself), so I think I know how to write them.
So I looked how XEN's hvmloader starts additional (virtual) processors inside a VM, and tried to port this code to GRUB2. See the code below.
The problem is that I reveice a triple fault when trying to "jump to protected mode", using a selector in the GDT I created before (the statement "ljmpl $0x08,$1f" generates the triple fault). Practically the same code works in XEN's hvmloader.
I think, meanwhile I understand what every statement of the assembler code does, but as I still get the triple fault, I think I still miss something.
Can someone help me further?
Here is the code of my grub module:
I think I understand, that the primary processor sends some APIC interprocessor interrupts to start additional processors (or cores). The additional CPU starts in real mode and has to use some "trampoline" to switch to protected mode.
I think I also understand that GRUB2 runs only on the primary processor. So I decided (again for the sole purpose of teaching myself) to write a grub module that starts additional CPUs, executes some little code on it and stop them again. I have already written user GRUB modules (again, to teach myself), so I think I know how to write them.
So I looked how XEN's hvmloader starts additional (virtual) processors inside a VM, and tried to port this code to GRUB2. See the code below.
The problem is that I reveice a triple fault when trying to "jump to protected mode", using a selector in the GDT I created before (the statement "ljmpl $0x08,$1f" generates the triple fault). Practically the same code works in XEN's hvmloader.
I think, meanwhile I understand what every statement of the assembler code does, but as I still get the triple fault, I think I still miss something.
Can someone help me further?
Here is the code of my grub module:
Code: Select all
#include <grub/dl.h>
#include <grub/mm.h>
#include <grub/err.h>
#include <grub/env.h>
#include <grub/misc.h>
#include <grub/file.h>
#include <grub/disk.h>
#include <grub/term.h>
#include <grub/loader.h>
#include <grub/command.h>
#include <grub/i18n.h>
GRUB_MOD_LICENSE ( "GPLv3+" ) ;
#define AP_BOOT_EIP 0x1000
#define wmb() barrier()
#define barrier() asm volatile ( "" : : : "memory" )
#define LAPIC_BASE_ADDRESS 0xfee00000
#define APIC_ICR 0x300
#define APIC_ICR_BUSY 0x01000
#define SET_APIC_DEST_FIELD(x) ((x)<<24)
#define LAPIC_ID(vcpu_id) ((vcpu_id) * 2)
#define APIC_ICR2 0x310
#define APIC_DM_INIT 0x00500
#define APIC_DM_STARTUP 0x00600
extern char ap_boot_start[], ap_boot_end[];
static int ap_callin, ap_cpuid;
asm (
" .text \n"
" .code16 \n"
"ap_boot_start: .code16 \n"
" mov %cs,%ax \n"
" mov %ax,%ds \n"
" lgdt gdt_desr-ap_boot_start\n"
" xor %ax, %ax \n"
" inc %ax \n"
" lmsw %ax \n"
" ljmpl $0x08,$1f \n"
"gdt_desr: \n"
" .word gdt_end - gdt - 1 \n"
" .long gdt \n"
"ap_boot_end: .code32 \n"
"1: mov $0x10,%eax \n"
" mov %eax,%ds \n"
" mov %eax,%es \n"
" mov %eax,%ss \n"
" movl $stack_top,%esp \n"
" movl %esp,%ebp \n"
" call ap_start \n"
"1: hlt \n"
" jmp 1b \n"
" \n"
" .align 8 \n"
"gdt: \n"
" .quad 0x0000000000000000 \n"
" .quad 0x00cf9a000000ffff \n" /* 0x08: Flat code segment */
" .quad 0x00cf92000000ffff \n" /* 0x10: Flat data segment */
"gdt_end: \n"
" \n"
" .bss \n"
" .align 8 \n"
"stack: \n"
" .skip 0x4000 \n"
"stack_top: \n"
" .text \n"
);
void ap_start(void); /* non-static avoids unused-function compiler warning */
/*static*/ void ap_start(void)
{
grub_printf(" - CPU%d ... \n", ap_cpuid);
/* cacheattr_init(); */
grub_printf("done.\n");
wmb();
ap_callin = 1;
}
static inline void cpu_relax(void)
{
asm volatile ( "rep ; nop" : : : "memory" );
}
grub_uint32_t lapic_read(grub_uint32_t reg)
{
return *(volatile grub_uint32_t *)(LAPIC_BASE_ADDRESS + reg);
}
void lapic_write(grub_uint32_t reg, grub_uint32_t val)
{
grub_printf ( "LAPIC write, reg = 0x%x, val = 0x%x\n" , reg , val ) ;
*(volatile grub_uint32_t *)(LAPIC_BASE_ADDRESS + reg) = val;
}
static void lapic_wait_ready(void)
{
while ( lapic_read(APIC_ICR) & APIC_ICR_BUSY )
cpu_relax();
}
static void boot_cpu(unsigned int cpu)
{
unsigned int icr2 = SET_APIC_DEST_FIELD(LAPIC_ID(cpu));
/* Initialise shared variables. */
ap_cpuid = cpu;
ap_callin = 0;
wmb();
/* Wake up the secondary processor: INIT-SIPI-SIPI... */
grub_printf ( "starting CPU %d\n" , cpu ) ;
lapic_wait_ready();
grub_printf ( "INIT\n" ) ;
lapic_write(APIC_ICR2, icr2);
lapic_write(APIC_ICR, APIC_DM_INIT);
lapic_wait_ready();
grub_printf ( "SIPI\n" ) ;
lapic_write(APIC_ICR2, icr2);
lapic_write(APIC_ICR, APIC_DM_STARTUP | (AP_BOOT_EIP >> 12));
lapic_wait_ready();
grub_printf ( "SIPI\n" ) ;
lapic_write(APIC_ICR2, icr2);
lapic_write(APIC_ICR, APIC_DM_STARTUP | (AP_BOOT_EIP >> 12));
lapic_wait_ready();
/*
* Wait for the secondary processor to complete initialisation.
* Do not touch shared resources meanwhile.
*/
while ( !ap_callin )
cpu_relax();
/* Take the secondary processor offline. */
lapic_write(APIC_ICR2, icr2);
lapic_write(APIC_ICR, APIC_DM_INIT);
lapic_wait_ready();
}
void smp_initialise(unsigned int nr_cpus)
{
unsigned int i;
unsigned char *pointer ;
grub_printf ( "startcode:\n" ) ;
for ( pointer = ap_boot_start ; pointer < (unsigned char *) ap_boot_end ; pointer ++ )
{
grub_printf ( "%02x " , *pointer ) ;
}
grub_printf ( "\n" ) ;
grub_printf ( "copying %d bytes of startcode\n" , ap_boot_end - ap_boot_start ) ;
grub_memcpy((void *)AP_BOOT_EIP, ap_boot_start, ap_boot_end - ap_boot_start);
grub_printf ( "verifying that startcode has been written...\n" ) ;
if ( grub_memcmp ( (void *) AP_BOOT_EIP , ap_boot_start , ap_boot_end - ap_boot_start ) != 0 )
{
grub_printf ( "FATAL ERROR: startcode was NOT properly written!\n" ) ;
grub_exit () ;
}
grub_printf("Multiprocessor initialisation:\n");
ap_start();
for ( i = 1; i < nr_cpus; i++ )
boot_cpu(i);
}
static grub_err_t
grub_smp ( struct grub_command *cmd __attribute__ ((unused)) ,
int argc , char *argv[] )
{
grub_printf ( "starting grub2 smp initialization module...\n" ) ;
smp_initialise ( 4 ) ;
}
static grub_command_t cmd_smp ;
GRUB_MOD_INIT ( smp )
{
cmd_smp = grub_register_command ( "smp" , grub_smp , 0 , N_ ( "Initialize smp" ) ) ;
}
GRUB_MOD_FINI ( smp )
{
grub_unregister_command ( cmd_smp ) ;
}