Skip to content

Commit

Permalink
Merge pull request #107 from Fly0307/opensbi-pr
Browse files Browse the repository at this point in the history 
🐞 fix(penglai-enclave-driver): free metadata, secure mem and pmp  issues#100
  • Loading branch information
@fengerhu1
fengerhu1 authored Dec 21, 2023
2 parents 9a5e2dc + 74f37d8 commit 7416e01
Show file tree
Hide file tree
Showing 7 changed files with 176 additions and 42 deletions.
27 changes: 17 additions & 10 deletions penglai-enclave-driver/penglai-enclave-driver.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@ MODULE_DESCRIPTION("enclave_ioctl");
MODULE_AUTHOR("LuXu");
MODULE_VERSION("enclave_ioctl");

// #define PENGLAI_DEBUG
#ifdef PENGLAI_DEBUG
#define dprint(...) printk(__VA_ARGS__)
#else
#define dprint(...)
#endif

static int enclave_mmap(struct file* f,struct vm_area_struct *vma)
{
return 0;
Expand Down Expand Up @@ -78,38 +85,38 @@ int enclave_ioctl_init(void)
void enclave_ioctl_exit(void)
{
unsigned long addr, order, count;
unsigned long *size = kmalloc(sizeof(unsigned long), GFP_KERNEL);
;
unsigned long *size_ptr = kmalloc(sizeof(unsigned long), GFP_KERNEL);

struct sbiret sbiret;
printk("enclave_ioctl_exit...\n");


sbiret = SBI_CALL_2(SBI_SM_FREE_ENCLAVE_MEM, __pa(size), FREE_MAX_MEMORY);
sbiret = SBI_CALL_2(SBI_SM_FREE_ENCLAVE_MEM, __pa(size_ptr), FREE_MAX_MEMORY);

addr = (unsigned long)(sbiret.value);
while (addr)
{
order = ilog2((*size) - 1) + 1;
order = ilog2((*size_ptr) - 1) + 1;
count = 0x1 << order;
if (count != (*size) && (*size > 0))
if (count != (*size_ptr) && (*size_ptr > 0))
{
printk("KERNEL MODULE: the number of free pages is not exponential times of two\n");
kfree(size);
kfree(size_ptr);
return;
}
printk("KERNEL MODULE: free secmem:paddr:%lx, vaddr:%lx, order:%lu\n", addr, __va(addr), order);
if ((*size) > 0)
if ((*size_ptr) > 0)
{
free_pages((long unsigned int)__va(addr), (order - RISCV_PGSHIFT));
}

sbiret = SBI_CALL_2(SBI_SM_FREE_ENCLAVE_MEM, __pa(size), FREE_MAX_MEMORY);
*size_ptr = 0;
sbiret = SBI_CALL_2(SBI_SM_FREE_ENCLAVE_MEM, __pa(size_ptr), FREE_MAX_MEMORY);

addr = (unsigned long)(sbiret.value);
}

deregister_device:
kfree(size);
kfree(size_ptr);
misc_deregister(&enclave_dev);
return;
}
Expand Down
35 changes: 31 additions & 4 deletions penglai-enclave-driver/penglai-enclave-elfloader.c
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,21 @@
#include "penglai-enclave-elfloader.h"
#define ROUND_TO(x, align) (((x) + ((align)-1)) & ~((align)-1))

// Function to print hex data
static void print_hex(const void *data, size_t size) {
const unsigned char *byte_data = data;
size_t i;

for (i = 0; i < size; ) {
printk("%02x%02x%02x%02x %02x%02x%02x%02x %02x%02x%02x%02x %02x%02x%02x%02x\n%02x%02x%02x%02x %02x%02x%02x%02x %02x%02x%02x%02x %02x%02x%02x%02x",
byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],
byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],
byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],
byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++]);
}
if (i % 16 != 0)
printk("\n");
}

int penglai_enclave_load_NOBITS_section(enclave_mem_t* enclave_mem, void * elf_sect_addr, int elf_sect_size)
{
Expand All @@ -25,8 +42,9 @@ int penglai_enclave_load_program(enclave_mem_t* enclave_mem, vaddr_t elf_prog_in
{
vaddr_t addr;
vaddr_t enclave_new_page;
vaddr_t begin_page = 0;
int size;
int r;
unsigned long r = 0;
for(addr = (vaddr_t)elf_prog_addr; addr < (vaddr_t)elf_prog_addr + elf_prog_size; addr += RISCV_PGSIZE)
{

Expand All @@ -35,8 +53,17 @@ int penglai_enclave_load_program(enclave_mem_t* enclave_mem, vaddr_t elf_prog_in
size = elf_prog_size % RISCV_PGSIZE;
else
size = RISCV_PGSIZE;
r = copy_from_user((void* )enclave_new_page, (void *)(elf_prog_infile_addr + addr - (vaddr_t)elf_prog_addr), size);
r += copy_from_user((void* )enclave_new_page, (void *)(elf_prog_infile_addr + addr - (vaddr_t)elf_prog_addr), size);
if (r)
{
printk("KERNEL MODULE: load_program copy_from_user failed r=0x%lx\n",r);
}

// dprint("[Penglai Driver@%s]copy segment 0x%lx to va:0x%lx -pa:0x%lx \n",__func__ ,(unsigned long)((void *)(elf_prog_addr + addr - (vaddr_t)elf_prog_addr)) ,(vaddr_t)enclave_new_page ,__pa((vaddr_t)enclave_new_page));
if(!begin_page)begin_page = enclave_new_page;
// print_hex(enclave_new_page, size);
}
dprint("[Penglai Driver@%s] SUCCESS! from 0x%lx to 0x%lx r=0x%x\n",__func__,(unsigned long)elf_prog_addr,(unsigned long)(elf_prog_addr + elf_prog_size) ,r);
return 0;
}

Expand Down Expand Up @@ -109,8 +136,8 @@ int penglai_enclave_loadelf(enclave_mem_t*enclave_mem, void* __user elf_ptr, uns
printk("KERNEL MODULE: penglai enclave load program failed\n");
return -1;
}
printk("[Penglai Driver@%s] elf_prog_addr:0x%lx elf_prog_size:0x%x, infile_addr:0x%lx", __func__,
elf_prog_addr, elf_prog_size, elf_prog_infile_addr);
// printk("[Penglai Driver@%s] elf_prog_addr:0x%lx elf_prog_size:0x%x, infile_addr:0x%lx", __func__,
// elf_prog_addr, elf_prog_size, elf_prog_infile_addr);
elf_prog_ptr += sizeof(struct elf_phdr);
}
return 0;
Expand Down
116 changes: 94 additions & 22 deletions penglai-enclave-driver/penglai-enclave-ioctl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ DEFINE_SPINLOCK(enclave_big_lock);
void acquire_big_lock(const char * str)
{
spin_lock(&enclave_big_lock);
printk("[PENGLAI Driver@%s] %s get lock\n", __func__, str);
dprint("[PENGLAI Driver@%s] %s get lock\n", __func__, str);
}

void release_big_lock(const char * str)
{
spin_unlock(&enclave_big_lock);
printk("[PENGLAI Driver@%s] %s release lock\n", __func__, str);
dprint("[PENGLAI Driver@%s] %s release lock\n", __func__, str);
}

unsigned int total_enclave_page(int elf_size, int stack_size)
Expand All @@ -28,7 +28,7 @@ unsigned int total_enclave_page(int elf_size, int stack_size)

int create_sbi_param(enclave_t* enclave, struct penglai_enclave_sbi_param * enclave_sbi_param,
unsigned long paddr, unsigned long size, unsigned long entry_point,
unsigned long untrusted_ptr, unsigned long untrusted_size, unsigned long free_mem)
unsigned long untrusted_ptr, unsigned long untrusted_ptr_paddr, unsigned long untrusted_size, unsigned long free_mem, unsigned long kbuffer_ptr, unsigned long kbuffer_paddr)
{
enclave_sbi_param -> eid_ptr = (unsigned int* )__pa(&enclave -> eid);
enclave_sbi_param -> ecall_arg0 = (unsigned long* )__pa(&enclave -> ocall_func_id);
Expand All @@ -39,10 +39,13 @@ int create_sbi_param(enclave_t* enclave, struct penglai_enclave_sbi_param * encl
enclave_sbi_param -> size = size;
enclave_sbi_param -> entry_point = entry_point;
enclave_sbi_param -> untrusted_ptr = untrusted_ptr;
enclave_sbi_param -> untrusted_size = untrusted_size;
enclave_sbi_param->untrusted_paddr = untrusted_ptr_paddr;
enclave_sbi_param->untrusted_size = untrusted_size;
enclave_sbi_param -> free_mem = free_mem;
//enclave share mem with kernel
enclave_sbi_param->kbuffer = ENCLAVE_DEFAULT_KBUFFER;
// enclave_sbi_param->kbuffer = ENCLAVE_DEFAULT_KBUFFER;
enclave_sbi_param->kbuffer = kbuffer_ptr;
enclave_sbi_param->kbuffer_paddr = kbuffer_paddr;
enclave_sbi_param->kbuffer_size = enclave->kbuffer_size;
return 0;
}
Expand Down Expand Up @@ -105,7 +108,7 @@ int penglai_enclave_create(struct file * filep, unsigned long args)
return -1;
}
long stack_size = enclave_param->stack_size; //DEFAULT_STACK_SIZE=1MB
long untrusted_mem_size = enclave_param->untrusted_mem_size; //DEFAULT_UNTRUSTED_SIZE=8KB
long untrusted_mem_size = enclave_param->untrusted_mem_size; //DEFAULT_UNTRUSTED_SIZE=8KB MAX=16MB
unsigned long untrusted_mem_ptr = enclave_param->untrusted_mem_ptr; //0
unsigned long kbuffer_ptr = ENCLAVE_DEFAULT_KBUFFER;
struct penglai_enclave_sbi_param *enclave_sbi_param = kmalloc(sizeof(struct penglai_enclave_sbi_param), GFP_KERNEL);
Expand All @@ -122,7 +125,7 @@ int penglai_enclave_create(struct file * filep, unsigned long args)
return -1;
}


acquire_big_lock(__func__);
enclave = create_enclave(total_pages); //May sleep
if(!enclave)
{
Expand Down Expand Up @@ -150,31 +153,65 @@ int penglai_enclave_create(struct file * filep, unsigned long args)
}
enclave->untrusted_mem->addr = (vaddr_t)untrusted_mem_ptr;
enclave->untrusted_mem->size = untrusted_mem_size;
printk("[Penglai Driver@%s] untrusted_mem->addr:0x%lx untrusted_mem->size:0x%lx\n",
__func__, (vaddr_t)untrusted_mem_ptr, untrusted_mem_size);
dprint("[Penglai Driver@%s] untrusted_mem->addr:0x%lx ,paddr:%lx, untrusted_mem->size:0x%lx\n",
__func__, (vaddr_t)untrusted_mem_ptr, __pa(untrusted_mem_ptr), untrusted_mem_size);

alloc_kbuffer(ENCLAVE_DEFAULT_KBUFFER_SIZE, &kbuffer_ptr, enclave); //May sleep
enclave->kbuffer = (vaddr_t)kbuffer_ptr;
enclave->kbuffer_size = ENCLAVE_DEFAULT_KBUFFER_SIZE;

free_mem = get_free_mem(&(enclave->enclave_mem->free_mem));
acquire_big_lock(__func__);
create_sbi_param(enclave, enclave_sbi_param,
(unsigned long)(enclave->enclave_mem->paddr),
enclave->enclave_mem->size, elf_entry, DEFAULT_UNTRUSTED_PTR,
untrusted_mem_size, __pa(free_mem));
(unsigned long)(enclave->enclave_mem->paddr),
enclave->enclave_mem->size, elf_entry, DEFAULT_UNTRUSTED_PTR, __pa(untrusted_mem_ptr),
untrusted_mem_size, __pa(free_mem), ENCLAVE_DEFAULT_KBUFFER, __pa(kbuffer_ptr));

printk("[Penglai Driver@%s] enclave_mem->paddr:0x%lx, size:0x%lx\n",
dprint("[Penglai Driver@%s] enclave_mem->paddr:0x%lx, size:0x%lx\n",
__func__, (unsigned long)(enclave->enclave_mem->paddr),
enclave->enclave_mem->size);

ret = SBI_CALL_1(SBI_SM_CREATE_ENCLAVE, __pa(enclave_sbi_param));
ret = SBI_CALL_2(SBI_SM_CREATE_ENCLAVE, __pa(enclave_sbi_param), false);

//if(ret < 0)
if(ret.error)
{
printk("KERNEL MODULE: SBI_SM_CREATE_ENCLAVE is failed \n");
goto destroy_enclave;
if(ret.value == ENCLAVE_NO_MEMORY){
//TODO: allocate certain memory region like sm_init
unsigned long addr;
int retry = 5;
addr = __get_free_pages(GFP_ATOMIC, DEFAULT_SECURE_PAGES_ORDER);
if(!addr)
{
printk("KERNEL MODULE: can not get free page which order is 0x%d", DEFAULT_SECURE_PAGES_ORDER);
goto destroy_enclave;
}
dprint("[Penglai Driver@%s] new alloc paddr:0x%lx\n",__func__, addr);
ret = SBI_CALL_2(SBI_SM_MEMORY_EXTEND, __pa(addr), (1 << (DEFAULT_SECURE_PAGES_ORDER + RISCV_PGSHIFT)) );

while (ret.value == RETRY_SPIN_LOCK && retry)
{
retry--;
ret = SBI_CALL_2(SBI_SM_MEMORY_EXTEND, __pa(addr), (1 << (DEFAULT_SECURE_PAGES_ORDER + RISCV_PGSHIFT)));
}

if(ret.error)
{
printk("KERNEL MODULE: sbi call extend memory is failed\n");
goto destroy_enclave;
}

ret = SBI_CALL_2(SBI_SM_CREATE_ENCLAVE, __pa(enclave_sbi_param), true);
if (ret.error)
{
printk("KERNEL MODULE: sbi call create enclave is failed\n");
goto destroy_enclave;
}

}
else{
goto destroy_enclave;
}
}

enclave_param->eid = enclave_idr_alloc(enclave);
Expand All @@ -183,7 +220,7 @@ int penglai_enclave_create(struct file * filep, unsigned long args)

release_big_lock(__func__);
kfree(enclave_sbi_param);

dprint("[Penglai Driver@%s] create enclave success!\t\n",__func__);
return ret.value;

destroy_enclave:
Expand Down Expand Up @@ -221,6 +258,8 @@ int penglai_enclave_destroy(struct file * filep, unsigned long args)
destroy_enclave(enclave);
enclave_idr_remove(eid);
} //otherwise, the run interfaces will destroy the enclave

// ret = SBI_CALL_1(SBI_SM_FREE_ENCLAVE_MEM,);
out:
release_big_lock(__func__);
return retval;
Expand Down Expand Up @@ -293,6 +332,7 @@ int penglai_enclave_run(struct file *filep, unsigned long args)
int retval = 0;
int resume_id = 0;

dprint("[Penglai Driver@%s] begin get_enclave_by_id\n", __func__);
acquire_big_lock(__func__);

enclave = get_enclave_by_id(eid);
Expand All @@ -307,7 +347,7 @@ int penglai_enclave_run(struct file *filep, unsigned long args)

enclave->is_running = 1; //set the flag

release_big_lock(__func__);


/* Use untrusted mem as in_out_buf*/
if((void*)untrusted_mem_ptr != NULL && untrusted_mem_size > 0){
Expand All @@ -316,11 +356,14 @@ int penglai_enclave_run(struct file *filep, unsigned long args)
return -EINVAL;
}
memset((void*)enclave->untrusted_mem->addr, 0, enclave->untrusted_mem->size);
if(copy_from_user((void*)enclave->untrusted_mem->addr, (void*)untrusted_mem_ptr, untrusted_mem_size))
if (copy_from_user((void *)enclave->untrusted_mem->addr, (void *)untrusted_mem_ptr, untrusted_mem_size))
{
printk("KERNEL MODULE: copy_from_user failed \n");
return -EFAULT;
}
}

printk("[Penglai Driver@%s] goto infinite run loop\n", __func__);
dprint("[Penglai Driver@%s] goto infinite run loop\n", __func__);
// In the (infinite loop), we do not need to acquire the lock
// The monitor is responsible to check the authentication
// It will only exit when either:
Expand All @@ -331,6 +374,7 @@ int penglai_enclave_run(struct file *filep, unsigned long args)
// the enclave (as resume_from_timer_irq will check status of an enclave)
ret = SBI_CALL_1(SBI_SM_RUN_ENCLAVE, enclave_eid);
resume_id = enclave->eid;
release_big_lock(__func__);

while((ret.value == ENCLAVE_TIMER_IRQ) || (ret.value == ENCLAVE_OCALL))
{
Expand Down Expand Up @@ -417,7 +461,8 @@ int penglai_enclave_attest(struct file * filep, unsigned long args)
enclave_t * enclave;
struct sbiret ret = {0};
int retval;

dprint("[Penglai Driver@%s] begin attest\n",
__func__);
acquire_big_lock(__func__);
enclave = get_enclave_by_id(eid);
if (!enclave)
Expand Down Expand Up @@ -547,7 +592,7 @@ int penglai_enclave_resume(struct file * filep, unsigned long args)
if (copy_to_user((void*)untrusted_mem_ptr, (void*)enclave->untrusted_mem->addr, ocall_buf_size))
return -EFAULT;
}
printk("[Penglai Driver@%s] return user for ocall \n", __func__);
dprint("[Penglai Driver@%s] return user for ocall,enclave->untrusted_mem:%lx\n\t kbuffer:%lx kbuffer_val:%s \n", __func__, (void *)(enclave->untrusted_mem->addr), enclave->kbuffer, (void *)(enclave->kbuffer));
return RETURN_USER_FOR_OCALL;
}
default:
Expand Down Expand Up @@ -595,6 +640,30 @@ int penglai_enclave_resume(struct file * filep, unsigned long args)
return retval;
}

int penglai_enclave_memory_reclaim(struct file * filep, unsigned long args){
int retval;
struct sbiret ret = {0};
struct mm_reclaim_arg_t* mm_reclaim;

struct penglai_enclave_user_param * enclave_param = (struct penglai_enclave_user_param*) args;
unsigned long eid = enclave_param ->eid;
enclave_t * enclave;

enclave = get_enclave_by_id(eid);

mm_reclaim = kmalloc(sizeof(struct mm_reclaim_arg_t), GFP_KERNEL);

acquire_big_lock(__func__);
ret = SBI_CALL_2(SBI_SM_MEMORY_RECLAIM, __pa(mm_reclaim), enclave->eid);
retval = ret.value;

release_big_lock(__func__);

dprint("[Penglai Driver@%s]A total of %lx enclave memory was reclaimed\n",__func__, mm_reclaim->resp_size);
kfree(mm_reclaim);
return retval;
}

long penglai_enclave_ioctl(struct file* filep, unsigned int cmd, unsigned long args)
{
char ioctl_data[1024];
Expand Down Expand Up @@ -634,6 +703,9 @@ long penglai_enclave_ioctl(struct file* filep, unsigned int cmd, unsigned long a
case PENGLAI_ENCLAVE_IOC_DESTROY_ENCLAVE:
ret = penglai_enclave_destroy(filep, (unsigned long)ioctl_data);
break;
case PENGLAI_ENCLAVE_MEMORY_RECLAIM:
ret = penglai_enclave_memory_reclaim(filep, (unsigned long)ioctl_data);
break;
case PENGLAI_ENCLAVE_IOC_DEBUG_PRINT:
sbiret = SBI_CALL_1(SBI_SM_DEBUG_PRINT, 0);
ret = sbiret.value;
Expand Down
Loading

0 comments on commit 7416e01

Please sign in to comment.