From 83495f3a7fd9284698065f348f9446398325fe42 Mon Sep 17 00:00:00 2001 From: Masamichi Takagi Date: Wed, 11 Sep 2019 12:30:55 +0900 Subject: [PATCH] ihklib: Support "all" in ihk_os_release_mem and ihk_release_mem Change-Id: I83b7a6f1cd6382082ca71496c7c890b744a8b3a4 --- linux/user/ihklib.c | 56 +++++++++++--- test/issues/1359/1359.c | 157 ++++++++++++++++++++++++++++++++++++++ test/issues/1359/Makefile | 40 ++++++++++ test/issues/1359/README | 9 +++ 4 files changed, 250 insertions(+), 12 deletions(-) create mode 100644 test/issues/1359/1359.c create mode 100644 test/issues/1359/Makefile create mode 100644 test/issues/1359/README diff --git a/linux/user/ihklib.c b/linux/user/ihklib.c index c9c92248..1d65d592 100644 --- a/linux/user/ihklib.c +++ b/linux/user/ihklib.c @@ -894,20 +894,28 @@ int ihk_release_mem(int index, struct ihk_mem_chunk* mem_chunks, int num_mem_chu int ret = 0, i, ret_ioctl; struct ihk_ioctl_mem_desc req = { 0 }; int fd = -1; + struct ihk_mem_chunk *query_mem_chunks = NULL; CHKANDJUMP(num_mem_chunks > IHK_MAX_NUM_MEM_CHUNKS, -EINVAL, "too many memory chunks specified\n"); - if ((fd = ihklib_device_open(index)) < 0) { - eprintf("%s: error: ihklib_device_open\n", - __func__); - ret = fd; - goto out; - } - CHKANDJUMP(!mem_chunks || !num_mem_chunks, -EINVAL, "invalid format\n"); + if (mem_chunks[0].size == IHK_SMP_MEM_ALL) { + /* Special case for releasing all memory */ + num_mem_chunks = ihk_get_num_reserved_mem_chunks(index); + query_mem_chunks = calloc(num_mem_chunks, + sizeof(struct ihk_mem_chunk)); + CHKANDJUMP(query_mem_chunks == NULL, -ENOMEM, + "failed to allocate query_mem_chunks\n"); + + ret = ihk_query_mem(index, query_mem_chunks, num_mem_chunks); + CHKANDJUMP(ret, -EINVAL, "ihk_query_mem failed\n"); + + mem_chunks = query_mem_chunks; + } + req.sizes = calloc(num_mem_chunks, sizeof(size_t)); if (!req.sizes) { eprintf("%s: error: allocating request sizes\n", @@ -930,6 +938,13 @@ int ihk_release_mem(int index, struct ihk_mem_chunk* mem_chunks, int num_mem_chu } req.num_chunks = num_mem_chunks; + if ((fd = ihklib_device_open(index)) < 0) { + eprintf("%s: error: ihklib_device_open\n", + __func__); + ret = fd; + goto out; + } + ret_ioctl = ioctl(fd, IHK_DEVICE_RELEASE_MEM, &req); CHKANDJUMP(ret_ioctl != 0, -errno, "ioctl failed"); @@ -937,6 +952,7 @@ int ihk_release_mem(int index, struct ihk_mem_chunk* mem_chunks, int num_mem_chu if (fd != -1) { close(fd); } + free(query_mem_chunks); free(req.sizes); free(req.numa_ids); return ret; @@ -1464,15 +1480,23 @@ int ihk_os_release_mem(int index, struct ihk_mem_chunk *mem_chunks, int ret = 0, i, ret_ioctl; struct ihk_ioctl_mem_desc req = { 0 }; int fd = -1; + struct ihk_mem_chunk *query_mem_chunks = NULL; CHKANDJUMP(num_mem_chunks > IHK_MAX_NUM_MEM_CHUNKS, -EINVAL, "too many memory chunks specified\n"); - if ((fd = ihklib_os_open(index)) < 0) { - eprintf("%s: error: ihklib_os_open\n", - __func__); - ret = fd; - goto out; + if (mem_chunks[0].size == IHK_SMP_MEM_ALL) { + /* Special case for releasing all memory */ + num_mem_chunks = ihk_os_get_num_assigned_mem_chunks(index); + query_mem_chunks = calloc(num_mem_chunks, + sizeof(struct ihk_mem_chunk)); + CHKANDJUMP(query_mem_chunks == NULL, -ENOMEM, + "failed to allocate query_mem_chunks\n"); + + ret = ihk_os_query_mem(index, query_mem_chunks, num_mem_chunks); + CHKANDJUMP(ret, -EINVAL, "ihk_os_query_mem failed\n"); + + mem_chunks = query_mem_chunks; } req.sizes = calloc(num_mem_chunks, sizeof(size_t)); @@ -1497,6 +1521,13 @@ int ihk_os_release_mem(int index, struct ihk_mem_chunk *mem_chunks, } req.num_chunks = num_mem_chunks; + if ((fd = ihklib_os_open(index)) < 0) { + eprintf("%s: error: ihklib_os_open\n", + __func__); + ret = fd; + goto out; + } + ret_ioctl = ioctl(fd, IHK_OS_RELEASE_MEM, &req); CHKANDJUMP(ret_ioctl != 0, -errno, "ioctl failed"); @@ -1504,6 +1535,7 @@ int ihk_os_release_mem(int index, struct ihk_mem_chunk *mem_chunks, if (fd != -1) { close(fd); } + free(query_mem_chunks); free(req.sizes); free(req.numa_ids); return ret; diff --git a/test/issues/1359/1359.c b/test/issues/1359/1359.c new file mode 100644 index 00000000..8ec7ec25 --- /dev/null +++ b/test/issues/1359/1359.c @@ -0,0 +1,157 @@ +#include +#include +#include +#include +#include +#include +#include + +#define Q(x) #x +#define QUOTE(x) Q(x) + +#define _OKNG(verb, cond, fmt, args...) do { \ + if (cond) { \ + if (verb) \ + printf("[OK] " fmt "\n", ##args); \ + } else { \ + printf("[NG] " fmt ": %d\n", ##args, ret); \ + goto out; \ + } \ +} while (0) + +#define OKNG(args...) _OKNG(1, ##args) +#define NG(args...) _OKNG(0, ##args) + +int main(int argc, char **argv) +{ + int ret = 0, status, i; + FILE *fp; + char buf[65536]; + size_t nread; + + char *prefix_usr = QUOTE(PREFIX_USR); + char *prefix_modules = QUOTE(PREFIX_MODULES); + char *prefix_images = QUOTE(PREFIX_IMAGES); + char cmd[1024]; + char fn[256]; + char kargs[256]; + + int cpus[3] = {12, 13, 14}; + int num_cpus = 3; + + struct ihk_mem_chunk mem_chunks[4]; + int num_mem_chunks; + + struct ihk_perf_event_attr perfattr[1] = { 0 }; + + printf("*** test start *************************\n"); + + sprintf(cmd, "taskset -c 0 insmod %s/ihk.ko", prefix_modules); + status = system(cmd); + NG(status == 0, "insmod ihk.ko"); + + sprintf(cmd, "taskset -c 0 insmod %s/ihk-smp-arm64.ko ihk_ikc_irq_core=0", prefix_modules); + status = system(cmd); + NG(status == 0, "insmod ihk-smp-arm64.ko"); + + // reserve mem 1G@4,1G@5 + num_mem_chunks = 2; + mem_chunks[0].size = (1UL << 30); + mem_chunks[0].numa_node_number = 4; + mem_chunks[1].size = (1UL << 30); + mem_chunks[1].numa_node_number = 5; + ret = ihk_reserve_mem(0, mem_chunks, num_mem_chunks); + OKNG(ret == 0, "ihk_reserve_mem"); + + // reserve cpus + ret = ihk_reserve_cpu(0, cpus, num_cpus); + OKNG(ret == 0, "ihk_reserve_cpu"); + + sprintf(cmd, "taskset -c 0 insmod %s/mcctrl.ko", prefix_modules); + status = system(cmd); + NG(status == 0, "insmod mcctrl.ko"); + + // create 0 + ret = ihk_create_os(0); + OKNG(ret == 0, "ihk_create_os"); + + // assign cpus + ret = ihk_os_assign_cpu(0, cpus, num_cpus); + OKNG(ret == 0, "ihk_os_assign_cpu"); + + // assign mem + ret = ihk_os_assign_mem(0, mem_chunks, num_mem_chunks); + OKNG(ret == 0, "ihk_os_assign_mem"); + + // load + sprintf(fn, "%s/mckernel.img", prefix_images); + ret = ihk_os_load(0, fn); + OKNG(ret == 0, "ihk_os_load"); + + // kargs + sprintf(kargs, "hidos dump_level=24 time_sharing"); + ret = ihk_os_kargs(0, kargs); + OKNG(ret == 0, "ihk_os_kargs"); + + // boot + ret = ihk_os_boot(0); + OKNG(ret == 0, "ihk_os_boot"); + + // status + ret = ihk_os_get_status(0); + for (i = 0; i < 5; i++) { + ret = ihk_os_get_status(0); + if (ret == IHK_STATUS_RUNNING) { + goto status_ok; + } + usleep(1000000); + } + + OKNG(0, "ihk_os_get_status"); + goto status_skip; + status_ok: + OKNG(1, "ihk_os_get_status"); + status_skip: + + // mcexec numactl + sprintf(cmd, "%s/bin/mcexec numactl -H", prefix_usr); + status = system(cmd); + OKNG(status == 0, "mcexec numactl"); + + // kmsg + sprintf(cmd, "%s/sbin/ihkosctl 0 kmsg", prefix_usr); + status = system(cmd); + OKNG(status == 0, "ihkosctl 0 kmsg"); + + ret = ihk_destroy_os(0, 0); + OKNG(ret == 0, "destroy"); + + // release cpus + ret = ihk_release_cpu(0, cpus, num_cpus); + OKNG(ret == 0, "ihk_release_cpu"); + + // release mem all + num_mem_chunks = 1; + mem_chunks[0].size = (unsigned long)-1; + mem_chunks[0].numa_node_number = 0; + ret = ihk_release_mem(0, mem_chunks, num_mem_chunks); + OKNG(ret == 0, "ihk_release_mem"); + + sprintf(cmd, "rmmod mcctrl"); + status = system(cmd); + OKNG(status == 0, "rmmod mcctrl"); + + sprintf(cmd, "rmmod ihk_smp_arm64"); + status = system(cmd); + NG(status == 0, "rmmod ihk_smp_arm64"); + + sprintf(cmd, "rmmod ihk"); + status = system(cmd); + NG(status == 0, "rmmod ihk"); + + printf("*** All tests finished\n"); + + ret = 0; +out: + return ret; +} diff --git a/test/issues/1359/Makefile b/test/issues/1359/Makefile new file mode 100644 index 00000000..ba3e63a3 --- /dev/null +++ b/test/issues/1359/Makefile @@ -0,0 +1,40 @@ +.SUFFIXES: # Clear suffixes +.SUFFIXES: .c + +CC = gcc + +LOCAL_INSTALL=YES + +ifeq ($(LOCAL_INSTALL), YES) +include $(HOME)/.mck_test_config.mk +PREFIX_USR=$(MCK_DIR) +PREFIX_MODULES=$(MCK_DIR)/kmod +PREFIX_IMAGES=$(MCK_DIR)/smp-arm64/kernel +else +UNAME_R=$(shell uname -r) +PREFIX_USR=/usr +PREFIX_MODULES=/lib/modules/$(UNAME_R)/extra/mckernel +PREFIX_IMAGES=/usr/share/mckernel/smp-arm64 +endif + +CPPFLAGS = -I$(PREFIX_USR)/include -DPREFIX_USR=$(PREFIX_USR) -DPREFIX_MODULES=$(PREFIX_MODULES) -DPREFIX_IMAGES=$(PREFIX_IMAGES) +CCFLAGS = -g +LDFLAGS = -L$(PREFIX_USR)/lib64 -lihk -Wl,-rpath -Wl,$(PREFIX_USR)/lib64 -lbfd +SRCS = $(shell ls *.c) +EXES = $(SRCS:.c=) +OBJS = $(SRCS:.c=.o) + +all: $(EXES) + +test: $(EXES) + sudo ./1359 + +%: %.o + $(CC) -o $@ $^ $(LDFLAGS) + +%.o: %.c + $(CC) $(CCFLAGS) $(CPPFLAGS) -c $< + +clean: + rm -f core $(EXES) $(OBJS) $(EXESMCK) $(OBJSMCK) + diff --git a/test/issues/1359/README b/test/issues/1359/README new file mode 100644 index 00000000..287a0920 --- /dev/null +++ b/test/issues/1359/README @@ -0,0 +1,9 @@ +=========== +How to test +=========== +(1) Make sure the following line in $HOME/.mck_test_config points to + the correct directory + + MCK_DIR= + +(2) make test