diff --git a/.github/labeler.yml b/.github/labeler.yml index de46af613f..0b674e7771 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -5,7 +5,7 @@ topic/data: - 'chapters/data/**/*' topic/compute: - - 'content/chapters/compute/**/*' + - 'chapters/compute/**/*' topic/io: - 'content/chapters/io/**/*' diff --git a/chapters/compute/Makefile b/chapters/compute/Makefile new file mode 100644 index 0000000000..80117e1ade --- /dev/null +++ b/chapters/compute/Makefile @@ -0,0 +1,37 @@ +# The script expect the source .svg files to be named as $TARGET-$i.svg, where $i is the frame number. +TARGETS = round-robin race-condition race-condition-toctou race-condition-lock +RVMD = reveal-md +MDPP = markdown-pp +FFMPEG = ffmpeg + +SLIDES ?= slides.mdpp +SLIDES_OUT ?= slides.md +MEDIA_DIR ?= generated-media +SITE ?= _site +OPEN ?= xdg-open + +.PHONY: all html clean videos + +all: videos html + +html: $(SITE) + +$(SITE): $(SLIDES) + $(MDPP) $< -o $(SLIDES_OUT) + $(RVMD) $(SLIDES_OUT) --static $@ + +videos: + mkdir -p $(MEDIA_DIR) + for TARGET in $(TARGETS); do \ + TARGET_DIR=$$(find -name $$TARGET -type d | grep media); \ + $(FFMPEG) -framerate 0.5 -f image2 -y \ + -i "$$TARGET_DIR/$$TARGET-%d.svg" -vf format=yuv420p $(MEDIA_DIR)/$$TARGET-generated.gif; \ + done + +open: $(SITE) + $(OPEN) $ $@ diff --git a/content/chapters/compute/lab/support/page-faults/page_faults.c b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/page_faults.c similarity index 99% rename from content/chapters/compute/lab/support/page-faults/page_faults.c rename to chapters/compute/copy-on-write/drills/tasks/page-faults/support/page_faults.c index aa4a67d705..1f1ba29d74 100644 --- a/content/chapters/compute/lab/support/page-faults/page_faults.c +++ b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/page_faults.c @@ -19,8 +19,6 @@ int main(void) struct stat filestat; char *data; - getchar(); - printf("Press enter to allocate 100 Bytes\n"); getchar(); diff --git a/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/log/CPPLINT.cfg b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/log/log.c b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/log/log.h b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/sock/sock_util.c b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/sock/sock_util.h b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/utils.h b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/README.md b/chapters/compute/copy-on-write/drills/tasks/shared-memory/README.md new file mode 100644 index 0000000000..3878cfcef9 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/README.md @@ -0,0 +1,25 @@ +# Shared Memory + +As you remember from the [Data chapter](reading/process-memory.md#memory-mapping), one way to allocate a given number of pages is to use the `mmap()` syscall. +Let's look at its [man page](https://man7.org/linux/man-pages/man2/mmap.2.html), specifically at the `flags` argument. +Its main purpose is to determine the way in which child processes interact with the mapped pages. + +[Quiz](../drills/questions/mmap-cow-flag.md) + +Now let's test this flag, as well as its opposite: `MAP_SHARED`. +Compile and run the code in `shared-memory/support/shared_memory.c`. + +1. See the value read by the parent is different from that written by the child. +Modify the `flags` parameter of `mmap()` so they are the same. + +1. Create a semaphore in the shared page and use it to make the parent signal the child before it can exit. +Use the API defined in [`semaphore.h`](https://man7.org/linux/man-pages/man0/semaphore.h.0p.html). + + **Be careful!** + The value written and read previously by the child and the parent, respectively, must not change. + + One way of creating a shared semaphore is to place it within a shared memory area, as we've just done. + This only works between "related" processes. + If you want to share a semaphore or other types of memory between any two processes, you need filesystem support. + For this, you should use **named semaphores**, created using [`sem_open()`](https://man7.org/linux/man-pages/man3/sem_open.3.html). + You'll get more accustomed to such functions in the [Application Interaction chapter]. diff --git a/content/chapters/compute/lab/solution/shared-memory/.gitignore b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/.gitignore similarity index 100% rename from content/chapters/compute/lab/solution/shared-memory/.gitignore rename to chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/.gitignore diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/Makefile b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/Makefile new file mode 100644 index 0000000000..2fae0479c5 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/Makefile @@ -0,0 +1,39 @@ +LDLIBS = -lpthread +BINARY = shared_memory + +all: $(BINARY) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARY): $(OBJS) $(LOGGER) + $(CC) $^ $(LDFLAGS) -o $@ $(LDLIBS) + +clean:: + -rm -f $(BINARIES) + +.PHONY: all clean diff --git a/content/chapters/compute/lab/solution/shared-memory/shared_memory.c b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/shared_memory.c similarity index 97% rename from content/chapters/compute/lab/solution/shared-memory/shared_memory.c rename to chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/shared_memory.c index fe1857ad68..fb089f1ed4 100644 --- a/content/chapters/compute/lab/solution/shared-memory/shared_memory.c +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/shared_memory.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/log/CPPLINT.cfg b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/log/log.c b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/log/log.h b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/sock/sock_util.c b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/sock/sock_util.h b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/utils.h b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/support/shared-memory/.gitignore b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/.gitignore similarity index 100% rename from content/chapters/compute/lab/support/shared-memory/.gitignore rename to chapters/compute/copy-on-write/drills/tasks/shared-memory/support/.gitignore diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/Makefile b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/Makefile new file mode 100644 index 0000000000..2fae0479c5 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/Makefile @@ -0,0 +1,39 @@ +LDLIBS = -lpthread +BINARY = shared_memory + +all: $(BINARY) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARY): $(OBJS) $(LOGGER) + $(CC) $^ $(LDFLAGS) -o $@ $(LDLIBS) + +clean:: + -rm -f $(BINARIES) + +.PHONY: all clean diff --git a/content/chapters/compute/lab/support/shared-memory/shared_memory.c b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/shared_memory.c similarity index 96% rename from content/chapters/compute/lab/support/shared-memory/shared_memory.c rename to chapters/compute/copy-on-write/drills/tasks/shared-memory/support/shared_memory.c index 917b393991..cc177048b3 100644 --- a/content/chapters/compute/lab/support/shared-memory/shared_memory.c +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/shared_memory.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/log/CPPLINT.cfg b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/log/log.c b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/log/log.h b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/sock/sock_util.c b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/sock/sock_util.h b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/utils.h b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/support/fork-faults/.gitignore b/chapters/compute/copy-on-write/guides/fork-faults/.gitignore similarity index 100% rename from content/chapters/compute/lab/support/fork-faults/.gitignore rename to chapters/compute/copy-on-write/guides/fork-faults/.gitignore diff --git a/chapters/compute/copy-on-write/guides/fork-faults/README.md b/chapters/compute/copy-on-write/guides/fork-faults/README.md new file mode 100644 index 0000000000..9482fdfab6 --- /dev/null +++ b/chapters/compute/copy-on-write/guides/fork-faults/README.md @@ -0,0 +1,30 @@ +# Fork Faults + +Now let's see the copy-on-write mechanism in practice. +Keep in mind that `fork()` is a function used to create a process. + +Open two terminals (or better: use [`tmux`](https://github.com/tmux/tmux/wiki)). +In one of them, compile and run the code in `fork-faults/support/fork_faults.c`. +After each time you press `Enter` in the first terminal window, run the following command in the second window: + +```console +student@os:~/.../fork-faults/support$ ps -o min_flt,maj_flt -p $(pidof fork_faults) +``` + +It will show you the number of minor and major page faults performed by the `fork_faults` process and its child. + +To better understand minor and major page faults, go through the [Minor and Major Page Faults](page-faults.md) exercise in Arena. + +[Quiz 1](../tasks/questions/parent-faults-before-fork.md) + +Note that after `fork()`-ing, there is a second row in the output of `ps`. +That corresponds to the child process. +The first one still corresponds to the parent. + +[Quiz 2](../tasks/questions/child-faults-after-write.md) + +Now it should be clear how demand paging differs from copy-on-write. +Shared memory is a similar concept. +It's a way of marking certain allocated pages so that copy-on-write is disabled. +As you may imagine, changes made by the parent to this memory are visible to the child and vice-versa. +You can learn more about it in [its dedicated section in the Arena](shared-memory). diff --git a/chapters/compute/copy-on-write/guides/fork-faults/support/Makefile b/chapters/compute/copy-on-write/guides/fork-faults/support/Makefile new file mode 100644 index 0000000000..1e76aa9529 --- /dev/null +++ b/chapters/compute/copy-on-write/guides/fork-faults/support/Makefile @@ -0,0 +1,38 @@ +BINARY = fork_faults + +all: $(BINARY) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARY): $(OBJS) $(LOGGER) + $(CC) $^ $(LDFLAGS) -o $@ $(LDLIBS) + +clean:: + -rm -f $(BINARY) + +.PHONY: all clean diff --git a/content/chapters/compute/lab/support/fork-faults/fork_faults.c b/chapters/compute/copy-on-write/guides/fork-faults/support/fork_faults.c similarity index 97% rename from content/chapters/compute/lab/support/fork-faults/fork_faults.c rename to chapters/compute/copy-on-write/guides/fork-faults/support/fork_faults.c index dc065a8cd2..4db7478e57 100644 --- a/content/chapters/compute/lab/support/fork-faults/fork_faults.c +++ b/chapters/compute/copy-on-write/guides/fork-faults/support/fork_faults.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/compute/copy-on-write/guides/fork-faults/support/utils/log/CPPLINT.cfg b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/copy-on-write/guides/fork-faults/support/utils/log/log.c b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/copy-on-write/guides/fork-faults/support/utils/log/log.h b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.c b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.h b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/copy-on-write/guides/fork-faults/support/utils/utils.h b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/media/copy-on-write-final.svg b/chapters/compute/copy-on-write/media/copy-on-write-final.svg similarity index 100% rename from content/chapters/compute/lab/media/copy-on-write-final.svg rename to chapters/compute/copy-on-write/media/copy-on-write-final.svg diff --git a/content/chapters/compute/lab/media/copy-on-write-initial.svg b/chapters/compute/copy-on-write/media/copy-on-write-initial.svg similarity index 100% rename from content/chapters/compute/lab/media/copy-on-write-initial.svg rename to chapters/compute/copy-on-write/media/copy-on-write-initial.svg diff --git a/content/chapters/compute/lab/content/copy-on-write.md b/chapters/compute/copy-on-write/reading/copy-on-write.md similarity index 57% rename from content/chapters/compute/lab/content/copy-on-write.md rename to chapters/compute/copy-on-write/reading/copy-on-write.md index 273b297ea0..efd5df9bac 100644 --- a/content/chapters/compute/lab/content/copy-on-write.md +++ b/chapters/compute/copy-on-write/reading/copy-on-write.md @@ -29,37 +29,6 @@ Then the process' page table points the page to the newly copied frame, as you c **Be careful!** Do not confuse **copy-on-write** with **demand paging**. -Remember from the [Data chapter](../../../data/) that **demand paging** means that when you allocate memory, the OS allocates virtual memory that remains unmapped to physical memory until it's used. +Remember from the [Data chapter](reading/working-with-memory.md) that **demand paging** means that when you allocate memory, the OS allocates virtual memory that remains unmapped to physical memory until it's used. On the other hand, **copy-on-write** posits that the virtual memory is already mapped to some frames. These frames are only duplicated when one of the processes attempts to write data to them. - -## Practice - -Now let's see the copy-on-write mechanism in practice. -Keep in mind that `fork()` is a function used to create a process. - -Open two terminals (or better: use [`tmux`](https://github.com/tmux/tmux/wiki)). -In one of them, compile and run the code in `support/fork-faults/fork_faults.c`. -After each time you press `Enter` in the first terminal window, run the following command in the second window: - -```console -student@os:~/.../lab/support/fork-faults$ ps -o min_flt,maj_flt -p $(pidof fork_faults) -``` - -It will show you the number of minor and major page faults performed by the `fork_faults` process and its child. - -To better understand minor and major page faults, go through the [Minor and Major Page Faults](arena.md#minor-and-major-page-faults) excercise in Arena. - -[Quiz 1](../quiz/parent-faults-before-fork.md) - -Note that after `fork()`-ing, there is a second row in the output of `ps`. -That corresponds to the child process. -The first one still corresponds to the parent. - -[Quiz 2](../quiz/child-faults-after-write.md) - -Now it should be clear how demand paging differs from copy-on-write. -Shared memory is a similar concept. -It's a way of marking certain allocated pages so that copy-on-write is disabled. -As you may imagine, changes made by the parent to this memory are visible to the child and vice-versa. -You can learn more about it in [its dedicated section in the Arena](./arena.md#shared-memory). diff --git a/content/chapters/compute/lecture/slides/copy-on-write.md b/chapters/compute/copy-on-write/slides/copy-on-write.md similarity index 83% rename from content/chapters/compute/lecture/slides/copy-on-write.md rename to chapters/compute/copy-on-write/slides/copy-on-write.md index 2a5adcbc9f..cd58f38568 100644 --- a/content/chapters/compute/lecture/slides/copy-on-write.md +++ b/chapters/compute/copy-on-write/slides/copy-on-write.md @@ -8,16 +8,16 @@ When `fork` returns, the child duplicates the contents of the parent's page table. -![Copy-on-write after fork()](./media/copy-on-write-initial.svg) +![Copy-on-write after fork()](../media/copy-on-write-initial.svg) ---- #### Copy-on-write * Writable pages are **copied and modified only** when written: `demo/copy-on-write/copy_on_write_overhead.c` -* [Quiz](../quiz/sections-always-shared.md) +* [Quiz](../drills/questions/sections-always-shared.md) -![Copy-on-write after Write](./media/copy-on-write-final.svg) +![Copy-on-write after Write](../media/copy-on-write-final.svg) ---- diff --git a/content/chapters/compute/lecture/slides/cool-extra-stuff.md b/chapters/compute/extra-slides.md similarity index 95% rename from content/chapters/compute/lecture/slides/cool-extra-stuff.md rename to chapters/compute/extra-slides.md index ee102b3f6f..5ef3b399b9 100644 --- a/content/chapters/compute/lecture/slides/cool-extra-stuff.md +++ b/chapters/compute/extra-slides.md @@ -16,7 +16,7 @@ #### Create a new process - `fork()` + `exec()` -* [Quiz](../quiz/exec-without-fork.md) +* [Quiz](../drills/questions/exec-without-fork.md) * "Save" the shell before calling `exec("/bin/ls")` * Use `fork()` and only the child calls `exec()` @@ -41,7 +41,7 @@ clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, c * Libc approach - `system`: -```console [1 | 2 | 3 | 5 | 7 | 9] +```console student@os:~/.../compute/lecture/demo/create-process$ strace -ff -e clone,execve ./system execve("./system", ["./system"], 0x7ffff3f20008 /* 60 vars */) = 0 clone(child_stack=0x7f3fa624aff0, flags=CLONE_VM|CLONE_VFORK|SIGCHLDstrace: Process 117770 attached @@ -67,7 +67,7 @@ strace: Process 117771 attached * Linux approach - `posix_spawn()` and `fork()` + `exec()`: -```console [1 | 2 | 3 | 5 | 7 | 8 | 9 | 11 | 13] +```console student@os:~/.../compute/lecture/demo/create-process$ strace -ff -e clone,execve ./posix_spawn execve("./posix_spawn", ["./posix_spawn"], 0x7ffcfed53928 /* 60 vars */) = 0 clone(child_stack=0x7f842691eff0, flags=CLONE_VM|CLONE_VFORK|SIGCHLDstrace: Process 118949 attached @@ -103,7 +103,7 @@ PS C:\...\lecture\demo\create-process> .\create_process.exe ### `fork()` vs `pthread_create()` -```console [1 - 2 | 3 - 4] +```console student@os:~/.../compute/lecture/demo/create-process$ strace -e clone ./fork_exec clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f7e83aa4810) = 5302 student@os:~/.../compute/lecture/demo/create-thread$ strace -e clone ./create_thread @@ -120,7 +120,7 @@ clone(child_stack=0x7f9ea7df0fb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGH ### The Suspended States -![Extended Thread Diagram](./media/thread-states-extended.svg) +![Extended Thread Diagram](../media/thread-states-extended.svg) ---- diff --git a/content/chapters/compute/lab/media/app-os-cpu-interaction.svg b/chapters/compute/hardware-perspective/media/app-os-cpu-interaction.svg similarity index 100% rename from content/chapters/compute/lab/media/app-os-cpu-interaction.svg rename to chapters/compute/hardware-perspective/media/app-os-cpu-interaction.svg diff --git a/content/chapters/compute/lab/content/hardware-perspective.md b/chapters/compute/hardware-perspective/reading/hardware-perspective.md similarity index 100% rename from content/chapters/compute/lab/content/hardware-perspective.md rename to chapters/compute/hardware-perspective/reading/hardware-perspective.md diff --git a/content/chapters/compute/lecture/demo/barrier/barrier.py b/chapters/compute/lecture-demos/barrier/barrier.py similarity index 100% rename from content/chapters/compute/lecture/demo/barrier/barrier.py rename to chapters/compute/lecture-demos/barrier/barrier.py diff --git a/content/chapters/compute/lecture/demo/condition/condition.py b/chapters/compute/lecture-demos/condition/condition.py similarity index 100% rename from content/chapters/compute/lecture/demo/condition/condition.py rename to chapters/compute/lecture-demos/condition/condition.py diff --git a/content/chapters/compute/lecture/demo/context-switch/.gitignore b/chapters/compute/lecture-demos/context-switch/.gitignore similarity index 100% rename from content/chapters/compute/lecture/demo/context-switch/.gitignore rename to chapters/compute/lecture-demos/context-switch/.gitignore diff --git a/content/chapters/compute/lecture/demo/context-switch/Makefile b/chapters/compute/lecture-demos/context-switch/Makefile similarity index 100% rename from content/chapters/compute/lecture/demo/context-switch/Makefile rename to chapters/compute/lecture-demos/context-switch/Makefile diff --git a/content/chapters/compute/lecture/demo/context-switch/cpu_bound.c b/chapters/compute/lecture-demos/context-switch/cpu_bound.c similarity index 100% rename from content/chapters/compute/lecture/demo/context-switch/cpu_bound.c rename to chapters/compute/lecture-demos/context-switch/cpu_bound.c diff --git a/content/chapters/compute/lecture/demo/context-switch/io_bound.c b/chapters/compute/lecture-demos/context-switch/io_bound.c similarity index 100% rename from content/chapters/compute/lecture/demo/context-switch/io_bound.c rename to chapters/compute/lecture-demos/context-switch/io_bound.c diff --git a/content/chapters/compute/lecture/demo/cooperative-scheduling/do.sh b/chapters/compute/lecture-demos/cooperative-scheduling/do.sh similarity index 100% rename from content/chapters/compute/lecture/demo/cooperative-scheduling/do.sh rename to chapters/compute/lecture-demos/cooperative-scheduling/do.sh diff --git a/content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/.config_coop b/chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/.config_coop similarity index 100% rename from content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/.config_coop rename to chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/.config_coop diff --git a/content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/.config_nocoop b/chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/.config_nocoop similarity index 100% rename from content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/.config_nocoop rename to chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/.config_nocoop diff --git a/content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/.gitignore b/chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/.gitignore similarity index 100% rename from content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/.gitignore rename to chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/.gitignore diff --git a/content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/Config.uk b/chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/Config.uk similarity index 100% rename from content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/Config.uk rename to chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/Config.uk diff --git a/content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/Makefile b/chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/Makefile similarity index 100% rename from content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/Makefile rename to chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/Makefile diff --git a/content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/Makefile.uk b/chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/Makefile.uk similarity index 100% rename from content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/Makefile.uk rename to chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/Makefile.uk diff --git a/content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/main.c b/chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/main.c similarity index 100% rename from content/chapters/compute/lecture/demo/cooperative-scheduling/workdir/apps/app-schedule-threads/main.c rename to chapters/compute/lecture-demos/cooperative-scheduling/workdir/apps/app-schedule-threads/main.c diff --git a/content/chapters/compute/lecture/demo/copy-on-write/.gitignore b/chapters/compute/lecture-demos/copy-on-write/.gitignore similarity index 100% rename from content/chapters/compute/lecture/demo/copy-on-write/.gitignore rename to chapters/compute/lecture-demos/copy-on-write/.gitignore diff --git a/content/chapters/compute/lecture/demo/copy-on-write/Makefile b/chapters/compute/lecture-demos/copy-on-write/Makefile similarity index 100% rename from content/chapters/compute/lecture/demo/copy-on-write/Makefile rename to chapters/compute/lecture-demos/copy-on-write/Makefile diff --git a/content/chapters/compute/lecture/demo/copy-on-write/copy_on_write_overhead.c b/chapters/compute/lecture-demos/copy-on-write/copy_on_write_overhead.c similarity index 100% rename from content/chapters/compute/lecture/demo/copy-on-write/copy_on_write_overhead.c rename to chapters/compute/lecture-demos/copy-on-write/copy_on_write_overhead.c diff --git a/content/chapters/compute/lecture/demo/create-process/.gitignore b/chapters/compute/lecture-demos/create-process/.gitignore similarity index 100% rename from content/chapters/compute/lecture/demo/create-process/.gitignore rename to chapters/compute/lecture-demos/create-process/.gitignore diff --git a/content/chapters/compute/lecture/demo/create-process/Makefile.linux b/chapters/compute/lecture-demos/create-process/Makefile.linux similarity index 100% rename from content/chapters/compute/lecture/demo/create-process/Makefile.linux rename to chapters/compute/lecture-demos/create-process/Makefile.linux diff --git a/content/chapters/compute/lecture/demo/create-process/Makefile.win b/chapters/compute/lecture-demos/create-process/Makefile.win similarity index 100% rename from content/chapters/compute/lecture/demo/create-process/Makefile.win rename to chapters/compute/lecture-demos/create-process/Makefile.win diff --git a/content/chapters/compute/lecture/demo/create-process/README.md b/chapters/compute/lecture-demos/create-process/README.md similarity index 97% rename from content/chapters/compute/lecture/demo/create-process/README.md rename to chapters/compute/lecture-demos/create-process/README.md index cc050aa5d8..7617bd8871 100644 --- a/content/chapters/compute/lecture/demo/create-process/README.md +++ b/chapters/compute/lecture-demos/create-process/README.md @@ -25,7 +25,7 @@ The Windows **syscall** `CreateProcess()` is quite similar to the Linux **librar It receives [(many) more arguments](https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessa). However, the main difference is that in order to run a specific command in the terminal, `CreateProcess()` must also be specified which terminal to use. In our example, we use the command prompt (`cmd`). -Hence, the first argumetn we give to `CreateProcess()` is `"cmd /s ls"`. +Hence, the first argument we give to `CreateProcess()` is `"cmd /s ls"`. ### `fork.c` diff --git a/content/chapters/compute/lecture/demo/create-process/create_process.c b/chapters/compute/lecture-demos/create-process/create_process.c similarity index 100% rename from content/chapters/compute/lecture/demo/create-process/create_process.c rename to chapters/compute/lecture-demos/create-process/create_process.c diff --git a/content/chapters/compute/lecture/demo/create-process/create_process.py b/chapters/compute/lecture-demos/create-process/create_process.py similarity index 100% rename from content/chapters/compute/lecture/demo/create-process/create_process.py rename to chapters/compute/lecture-demos/create-process/create_process.py diff --git a/content/chapters/compute/lecture/demo/create-process/fork.c b/chapters/compute/lecture-demos/create-process/fork.c similarity index 100% rename from content/chapters/compute/lecture/demo/create-process/fork.c rename to chapters/compute/lecture-demos/create-process/fork.c diff --git a/content/chapters/compute/lecture/demo/create-process/fork_exec.c b/chapters/compute/lecture-demos/create-process/fork_exec.c similarity index 100% rename from content/chapters/compute/lecture/demo/create-process/fork_exec.c rename to chapters/compute/lecture-demos/create-process/fork_exec.c diff --git a/content/chapters/compute/lecture/demo/create-process/multiple_forks.c b/chapters/compute/lecture-demos/create-process/multiple_forks.c similarity index 100% rename from content/chapters/compute/lecture/demo/create-process/multiple_forks.c rename to chapters/compute/lecture-demos/create-process/multiple_forks.c diff --git a/content/chapters/compute/lecture/demo/create-process/posix_spawn.c b/chapters/compute/lecture-demos/create-process/posix_spawn.c similarity index 100% rename from content/chapters/compute/lecture/demo/create-process/posix_spawn.c rename to chapters/compute/lecture-demos/create-process/posix_spawn.c diff --git a/content/chapters/compute/lecture/demo/create-process/system.c b/chapters/compute/lecture-demos/create-process/system.c similarity index 100% rename from content/chapters/compute/lecture/demo/create-process/system.c rename to chapters/compute/lecture-demos/create-process/system.c diff --git a/content/chapters/compute/lecture/demo/create-thread/.gitignore b/chapters/compute/lecture-demos/create-thread/.gitignore similarity index 100% rename from content/chapters/compute/lecture/demo/create-thread/.gitignore rename to chapters/compute/lecture-demos/create-thread/.gitignore diff --git a/content/chapters/compute/lecture/demo/create-thread/Makefile b/chapters/compute/lecture-demos/create-thread/Makefile similarity index 100% rename from content/chapters/compute/lecture/demo/create-thread/Makefile rename to chapters/compute/lecture-demos/create-thread/Makefile diff --git a/content/chapters/compute/lecture/demo/create-thread/create_thread.c b/chapters/compute/lecture-demos/create-thread/create_thread.c similarity index 100% rename from content/chapters/compute/lecture/demo/create-thread/create_thread.c rename to chapters/compute/lecture-demos/create-thread/create_thread.c diff --git a/content/chapters/compute/lecture/demo/create-thread/create_thread.py b/chapters/compute/lecture-demos/create-thread/create_thread.py similarity index 100% rename from content/chapters/compute/lecture/demo/create-thread/create_thread.py rename to chapters/compute/lecture-demos/create-thread/create_thread.py diff --git a/content/chapters/compute/lecture/demo/deadlock/deadlock.py b/chapters/compute/lecture-demos/deadlock/deadlock.py similarity index 100% rename from content/chapters/compute/lecture/demo/deadlock/deadlock.py rename to chapters/compute/lecture-demos/deadlock/deadlock.py diff --git a/content/chapters/compute/lecture/demo/granularity/.gitignore b/chapters/compute/lecture-demos/granularity/.gitignore similarity index 100% rename from content/chapters/compute/lecture/demo/granularity/.gitignore rename to chapters/compute/lecture-demos/granularity/.gitignore diff --git a/content/chapters/compute/lecture/demo/granularity/Makefile.coarse b/chapters/compute/lecture-demos/granularity/Makefile.coarse similarity index 100% rename from content/chapters/compute/lecture/demo/granularity/Makefile.coarse rename to chapters/compute/lecture-demos/granularity/Makefile.coarse diff --git a/content/chapters/compute/lecture/demo/granularity/Makefile.fine b/chapters/compute/lecture-demos/granularity/Makefile.fine similarity index 100% rename from content/chapters/compute/lecture/demo/granularity/Makefile.fine rename to chapters/compute/lecture-demos/granularity/Makefile.fine diff --git a/content/chapters/compute/lecture/demo/granularity/granularity.c b/chapters/compute/lecture-demos/granularity/granularity.c similarity index 100% rename from content/chapters/compute/lecture/demo/granularity/granularity.c rename to chapters/compute/lecture-demos/granularity/granularity.c diff --git a/content/chapters/compute/lecture/demo/race-condition/.gitignore b/chapters/compute/lecture-demos/race-condition/.gitignore similarity index 100% rename from content/chapters/compute/lecture/demo/race-condition/.gitignore rename to chapters/compute/lecture-demos/race-condition/.gitignore diff --git a/content/chapters/compute/lecture/demo/race-condition/Makefile b/chapters/compute/lecture-demos/race-condition/Makefile similarity index 100% rename from content/chapters/compute/lecture/demo/race-condition/Makefile rename to chapters/compute/lecture-demos/race-condition/Makefile diff --git a/content/chapters/compute/lecture/demo/race-condition/race_condition.c b/chapters/compute/lecture-demos/race-condition/race_condition.c similarity index 100% rename from content/chapters/compute/lecture/demo/race-condition/race_condition.c rename to chapters/compute/lecture-demos/race-condition/race_condition.c diff --git a/content/chapters/compute/lecture/demo/race-condition/race_condition_atomic_assembly.c b/chapters/compute/lecture-demos/race-condition/race_condition_atomic_assembly.c similarity index 100% rename from content/chapters/compute/lecture/demo/race-condition/race_condition_atomic_assembly.c rename to chapters/compute/lecture-demos/race-condition/race_condition_atomic_assembly.c diff --git a/content/chapters/compute/lecture/demo/race-condition/race_condition_atomic_gcc.c b/chapters/compute/lecture-demos/race-condition/race_condition_atomic_gcc.c similarity index 100% rename from content/chapters/compute/lecture/demo/race-condition/race_condition_atomic_gcc.c rename to chapters/compute/lecture-demos/race-condition/race_condition_atomic_gcc.c diff --git a/content/chapters/compute/lecture/demo/race-condition/race_condition_mutex.c b/chapters/compute/lecture-demos/race-condition/race_condition_mutex.c similarity index 100% rename from content/chapters/compute/lecture/demo/race-condition/race_condition_mutex.c rename to chapters/compute/lecture-demos/race-condition/race_condition_mutex.c diff --git a/content/chapters/compute/lecture/demo/race-condition/race_condition_spinlock.c b/chapters/compute/lecture-demos/race-condition/race_condition_spinlock.c similarity index 100% rename from content/chapters/compute/lecture/demo/race-condition/race_condition_spinlock.c rename to chapters/compute/lecture-demos/race-condition/race_condition_spinlock.c diff --git a/content/chapters/compute/lecture/demo/utils/get_time.h b/chapters/compute/lecture-demos/utils/get_time.h similarity index 100% rename from content/chapters/compute/lecture/demo/utils/get_time.h rename to chapters/compute/lecture-demos/utils/get_time.h diff --git a/content/chapters/compute/lecture/demo/utils/utils.h b/chapters/compute/lecture-demos/utils/utils.h similarity index 100% rename from content/chapters/compute/lecture/demo/utils/utils.h rename to chapters/compute/lecture-demos/utils/utils.h diff --git a/content/chapters/compute/lecture/media/cpu-internals.jpg b/chapters/compute/overview/media/cpu-internals.jpg similarity index 100% rename from content/chapters/compute/lecture/media/cpu-internals.jpg rename to chapters/compute/overview/media/cpu-internals.jpg diff --git a/chapters/compute/overview/reading/overview.md b/chapters/compute/overview/reading/overview.md new file mode 100644 index 0000000000..62c7fd09eb --- /dev/null +++ b/chapters/compute/overview/reading/overview.md @@ -0,0 +1,11 @@ +# Compute + +## Contents + +- [Hardware Perspective](../../hardware-perspective/reading/hardware-perspective.md) +- [Processes](../../processes/reading/processes.md) +- [Threads](../../threads/reading/threads.md) +- [Processes and Threads in `apache2`](../../processes-threads-apache2/reading/processes-threads-apache2.md) +- [Copy-on-Write](../../copy-on-write/reading/copy-on-write.md) +- [Synchronization](../../synchronization/reading/synchronization.md) +- [User Level Threads](../../user-level-threads/reading/user-level-threads.md) diff --git a/content/chapters/compute/lecture/slides/intro.md b/chapters/compute/overview/slides/intro.md similarity index 96% rename from content/chapters/compute/lecture/slides/intro.md rename to chapters/compute/overview/slides/intro.md index fff753f1f3..70f95cd700 100644 --- a/content/chapters/compute/lecture/slides/intro.md +++ b/chapters/compute/overview/slides/intro.md @@ -23,7 +23,7 @@ How? #### Some CPU internals -![CPU Internals](media/cpu-internals.jpg) +![CPU Internals](../media/cpu-internals.jpg) * Multiple **cores** per CPU * Still not enough for all processes diff --git a/content/chapters/compute/lab/quiz/apache2-strace.md b/chapters/compute/processes-threads-apache2/drills/questions/apache2-strace.md similarity index 100% rename from content/chapters/compute/lab/quiz/apache2-strace.md rename to chapters/compute/processes-threads-apache2/drills/questions/apache2-strace.md diff --git a/chapters/compute/processes-threads-apache2/drills/tasks/apache2/README.md b/chapters/compute/processes-threads-apache2/drills/tasks/apache2/README.md new file mode 100644 index 0000000000..5cba42122d --- /dev/null +++ b/chapters/compute/processes-threads-apache2/drills/tasks/apache2/README.md @@ -0,0 +1,17 @@ +# Investigate `apache2` Using `strace` + +Enter the `apache2/support/` folder and go through the practice items below. + +1. Use `strace` to discover the server document root. +The document root is the path in the filesystem from where `httpd` serves all the files requested by the clients. + +First, you will have to stop the running container using `make stop`, then restart it with `make run-privileged`. + +1. Use `strace` inside the container to attach to the worker processes (use the `-p` option for this). +You will also have to use the `-f` flag with `strace`, so that it will follow all the threads inside the processes. + +1. After you have attached successfully to all worker processes, use the `curl` command to send a request. + +1. Then check the `strace` output to see what files were opened by the server. + +[Quiz](../../questions/apache2-strace.md) diff --git a/content/chapters/compute/lab/support/apache2/Dockerfile b/chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/Dockerfile similarity index 81% rename from content/chapters/compute/lab/support/apache2/Dockerfile rename to chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/Dockerfile index 76ebee9ca9..4842e7a7af 100644 --- a/content/chapters/compute/lab/support/apache2/Dockerfile +++ b/chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/Dockerfile @@ -1,7 +1,7 @@ FROM httpd:2.4.54 -RUN apt-get -y update && apt-get -y install procps strace net-tools +RUN apt-get -y update && apt-get -y install procps strace net-tools \ + && rm -rf /var/lib/apt/lists/* COPY httpd-mpm.conf /usr/local/apache2/conf/extra COPY httpd.conf /usr/local/apache2/conf/ - diff --git a/content/chapters/compute/lab/support/apache2/Makefile b/chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/Makefile similarity index 100% rename from content/chapters/compute/lab/support/apache2/Makefile rename to chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/Makefile diff --git a/content/chapters/compute/lab/support/apache2/httpd-mpm.conf b/chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/httpd-mpm.conf similarity index 100% rename from content/chapters/compute/lab/support/apache2/httpd-mpm.conf rename to chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/httpd-mpm.conf diff --git a/content/chapters/compute/lab/support/apache2/httpd.conf b/chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/httpd.conf similarity index 99% rename from content/chapters/compute/lab/support/apache2/httpd.conf rename to chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/httpd.conf index 802cd74f3c..284f8f85d3 100644 --- a/content/chapters/compute/lab/support/apache2/httpd.conf +++ b/chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/httpd.conf @@ -470,7 +470,7 @@ LogLevel warn # 1) plain text 2) local redirects 3) external redirects # # Some examples: -#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 500 "The server took a hit and needs to rest." #ErrorDocument 404 /missing.html #ErrorDocument 404 "/cgi-bin/missing_handler.pl" #ErrorDocument 402 http://www.example.com/subscription_info.html @@ -540,7 +540,7 @@ Include conf/extra/proxy-html.conf # Secure (SSL/TLS) connections #Include conf/extra/httpd-ssl.conf # -# Note: The following must must be present to support +# Note: The following must be present to support # starting without SSL on platforms with no /dev/random equivalent # but a statically compiled-in mod_ssl. # diff --git a/content/chapters/compute/lab/support/apache2/make_conn.py b/chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/make_conn.py similarity index 100% rename from content/chapters/compute/lab/support/apache2/make_conn.py rename to chapters/compute/processes-threads-apache2/drills/tasks/apache2/support/make_conn.py diff --git a/content/chapters/compute/lab/content/processes-threads-apache2.md b/chapters/compute/processes-threads-apache2/guides/apache2/README.md similarity index 57% rename from content/chapters/compute/lab/content/processes-threads-apache2.md rename to chapters/compute/processes-threads-apache2/guides/apache2/README.md index 89500b96a1..65c6ee5edd 100644 --- a/content/chapters/compute/lab/content/processes-threads-apache2.md +++ b/chapters/compute/processes-threads-apache2/guides/apache2/README.md @@ -1,27 +1,7 @@ -# Usage of Processes and Threads in `apache2` - -We'll take a look at how a real-world application - the `apache2` HTTP server - makes use of processes and threads. -Since the server must be able to handle multiple clients at the same time, it must therefore use some form of concurrency. -When a new client arrives, the server offloads the work of interacting with that client to another process or thread. - -The choice of whether to use multiple processes or threads is not baked into the code. -Instead, `apache2` provides a couple of modules called MPMs (Multi-Processing Modules). -Each module implements a different concurrency model, and the users can pick whatever module best fits their needs by editing the server configuration files. - -The most common MPMs are - -- `prefork`: there are multiple worker processes, each process is single-threaded and handles one client request at a time -- `worker`: there are multiple worker processes, each process is multi-threaded, and each thread handles one client request at a time -- `event`: same as `worker` but designed to better handle some particular use cases - -In principle, `prefork` provides more stability and backwards compatibility, but it has a bigger overhead. -On the other hand, `worker` and `event` are more scalable, and thus able to handle more simultaneous connections, due to the usage of threads. -On modern systems, `event` is almost always the default. - -## `apache2` Live Action +# `apache2` Live Action Let's run an actual instance of `apache2` and see how everything works. -Go to `support/apache2` and run `make run`. +Go to `apache2/support` and run `make run`. This will start a container with `apache2` running inside. Check that the server runs as expected: @@ -34,7 +14,7 @@ student@os:~$ curl localhost:8080 Now go inside the container and take a look at running processes: ```console -student@os:~/.../lab/support/apache2$ docker exec -it apache2-test bash +student@os:~/.../apache2/support$ docker exec -it apache2-test bash root@56b9a761d598:/usr/local/apache2# ps -ef UID PID PPID C STIME TTY TIME CMD @@ -90,10 +70,10 @@ If all the threads are busy, then the server will spawn more worker processes (a Let's see this dynamic scaling in action. We need to create a number of simultaneous connections that is larger than the current number of threads. -There is a simple script in `support/apache2/make_conn.py` to do this: +There is a simple script in `/apache2/supportmake_conn.py` to do this: ```console -student@os:~/.../lab/support/apache2$ python3 make_conn.py localhost 8080 +student@os:~/.../apache2/support$ python3 make_conn.py localhost 8080 Press ENTER to exit ``` @@ -102,7 +82,7 @@ The script has created 100 connections and will keep them open until we press En Now, in another terminal, let's check the situation inside the container: ```console -student@os:~/.../lab/support/apache2$ docker exec -it apache2-test bash +student@os:~/.../apache2/support$ docker exec -it apache2-test bash root@56b9a761d598:/usr/local/apache2# ps -efL UID PID PPID LWP C NLWP STIME TTY TIME CMD @@ -134,40 +114,3 @@ root 152 146 152 0 1 21:10 pts/1 00:00:00 ps -efL ``` We see a much larger number of threads, as expected. - -## Practice: Investigate `apache2` Using `strace` - -Use `strace` to discover the server document root. -The document root is the path in the filesystem from where `httpd` serves all the files requested by the clients. - -First, you will have to stop the running container using `make stop`, then restart it with `make run-privileged`. - -Then you will use `strace` inside the container to attach to the worker processes (use the `-p` option for this). -You will also have to use the `-f` flag with `strace`, so that it will follow all the threads inside the processes. - -After you have attached successfully to all worker processes, use the `curl` command to send a request, like the one in the beginning of this section. - -Then check the `strace` output to see what files were opened by the server. - -[Quiz](../quiz/apache2-strace.md) - -## Conclusion - -So far, you've probably seen that spawning a process can "use" a different program (hence the path in the args of `system` or `Popen`), but some languages such as Python allow you to spawn a process that executes a function from the same script. -A thread, however, can only start from a certain entry point **within the current address space**, as it is bound to the same process. -Concretely, a process is but a group of threads. -For this reason, when we talk about scheduling or synchronization, we talk about threads. -A thread is, thus, an abstraction of a task running on a CPU core. -A process is a logical group of such tasks. - -We can sum up what we've learned so far by saying that processes are better used for separate, independent work, such as the different connections handled by a server. -Conversely, threads are better suited for replicated work: when the same task has to be performed on multiple cores. -However, replicated work can also be suited for processes. -Distributed applications, however, leverage different processes as this allows them to run on multiple physical machines at once. -This is required by the very large workloads such applications are commonly required to process. - -These rules are not set in stone, though. -Like we saw in the `apache2` example, the server uses multiple threads as well as multiple processes. -This provides a degree of stability - if one worker thread crashes, it will only crash the other threads belonging to the same process - while still taking advantage of the light resource usage inherent to threads. - -These kinds of trade-offs are a normal part of the development of real-world applications. diff --git a/chapters/compute/processes-threads-apache2/guides/apache2/support/Dockerfile b/chapters/compute/processes-threads-apache2/guides/apache2/support/Dockerfile new file mode 100644 index 0000000000..4842e7a7af --- /dev/null +++ b/chapters/compute/processes-threads-apache2/guides/apache2/support/Dockerfile @@ -0,0 +1,7 @@ +FROM httpd:2.4.54 + +RUN apt-get -y update && apt-get -y install procps strace net-tools \ + && rm -rf /var/lib/apt/lists/* + +COPY httpd-mpm.conf /usr/local/apache2/conf/extra +COPY httpd.conf /usr/local/apache2/conf/ diff --git a/chapters/compute/processes-threads-apache2/guides/apache2/support/Makefile b/chapters/compute/processes-threads-apache2/guides/apache2/support/Makefile new file mode 100644 index 0000000000..52b73af4e4 --- /dev/null +++ b/chapters/compute/processes-threads-apache2/guides/apache2/support/Makefile @@ -0,0 +1,10 @@ +run: + docker build -t apache2-test . + docker run --name apache2-test --rm -dit -p 8080:80 apache2-test + +run-privileged: + docker build -t apache2-test . + docker run --privileged --name apache2-test --rm -dit -p 8080:80 apache2-test + +stop: + docker stop apache2-test diff --git a/chapters/compute/processes-threads-apache2/guides/apache2/support/httpd-mpm.conf b/chapters/compute/processes-threads-apache2/guides/apache2/support/httpd-mpm.conf new file mode 100644 index 0000000000..bcbc654fcc --- /dev/null +++ b/chapters/compute/processes-threads-apache2/guides/apache2/support/httpd-mpm.conf @@ -0,0 +1,119 @@ +# +# Server-Pool Management (MPM specific) +# + +# +# PidFile: The file in which the server should record its process +# identification number when it starts. +# +# Note that this is the default PidFile for most MPMs. +# + + PidFile "logs/httpd.pid" + + +# +# Only one of the below sections will be relevant on your +# installed httpd. Use "apachectl -l" to find out the +# active mpm. +# + +# prefork MPM +# StartServers: number of server processes to start +# MinSpareServers: minimum number of server processes which are kept spare +# MaxSpareServers: maximum number of server processes which are kept spare +# MaxRequestWorkers: maximum number of server processes allowed to start +# MaxConnectionsPerChild: maximum number of connections a server process serves +# before terminating + + StartServers 5 + MinSpareServers 5 + MaxSpareServers 10 + MaxRequestWorkers 250 + MaxConnectionsPerChild 0 + + +# worker MPM +# StartServers: initial number of server processes to start +# MinSpareThreads: minimum number of worker threads which are kept spare +# MaxSpareThreads: maximum number of worker threads which are kept spare +# ThreadsPerChild: constant number of worker threads in each server process +# MaxRequestWorkers: maximum number of worker threads +# MaxConnectionsPerChild: maximum number of connections a server process serves +# before terminating + + StartServers 3 + MinSpareThreads 75 + MaxSpareThreads 250 + ThreadsPerChild 25 + MaxRequestWorkers 400 + MaxConnectionsPerChild 0 + + +# event MPM +# StartServers: initial number of server processes to start +# MinSpareThreads: minimum number of worker threads which are kept spare +# MaxSpareThreads: maximum number of worker threads which are kept spare +# ThreadsPerChild: constant number of worker threads in each server process +# MaxRequestWorkers: maximum number of worker threads +# MaxConnectionsPerChild: maximum number of connections a server process serves +# before terminating + + StartServers 2 + MinSpareThreads 10 + MaxSpareThreads 60 + ThreadsPerChild 5 + MaxRequestWorkers 80 + MaxConnectionsPerChild 0 + + +# NetWare MPM +# ThreadStackSize: Stack size allocated for each worker thread +# StartThreads: Number of worker threads launched at server startup +# MinSpareThreads: Minimum number of idle threads, to handle request spikes +# MaxSpareThreads: Maximum number of idle threads +# MaxThreads: Maximum number of worker threads alive at the same time +# MaxConnectionsPerChild: Maximum number of connections a thread serves. It +# is recommended that the default value of 0 be set +# for this directive on NetWare. This will allow the +# thread to continue to service requests indefinitely. + + ThreadStackSize 65536 + StartThreads 250 + MinSpareThreads 25 + MaxSpareThreads 250 + MaxThreads 1000 + MaxConnectionsPerChild 0 + + +# OS/2 MPM +# StartServers: Number of server processes to maintain +# MinSpareThreads: Minimum number of idle threads per process, +# to handle request spikes +# MaxSpareThreads: Maximum number of idle threads per process +# MaxConnectionsPerChild: Maximum number of connections per server process + + StartServers 2 + MinSpareThreads 5 + MaxSpareThreads 10 + MaxConnectionsPerChild 0 + + +# WinNT MPM +# ThreadsPerChild: constant number of worker threads in the server process +# MaxConnectionsPerChild: maximum number of connections a server process serves + + ThreadsPerChild 150 + MaxConnectionsPerChild 0 + + +# The maximum number of free Kbytes that every allocator is allowed +# to hold without calling free(). In threaded MPMs, every thread has its own +# allocator. When not set, or when set to zero, the threshold will be set to +# unlimited. + + MaxMemFree 2048 + + + MaxMemFree 100 + diff --git a/chapters/compute/processes-threads-apache2/guides/apache2/support/httpd.conf b/chapters/compute/processes-threads-apache2/guides/apache2/support/httpd.conf new file mode 100644 index 0000000000..6045ad57bc --- /dev/null +++ b/chapters/compute/processes-threads-apache2/guides/apache2/support/httpd.conf @@ -0,0 +1,551 @@ +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so "logs/access_log" +# with ServerRoot set to "/usr/local/apache2" will be interpreted by the +# server as "/usr/local/apache2/logs/access_log", whereas "/logs/access_log" +# will be interpreted as '/logs/access_log'. + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to specify a local disk on the +# Mutex directive, if file-based mutexes are used. If you wish to share the +# same ServerRoot for multiple httpd daemons, you will need to change at +# least PidFile. +# +ServerRoot "/usr/local/apache2" + +# +# Mutex: Allows you to set the mutex mechanism and mutex file directory +# for individual mutexes, or change the global defaults +# +# Uncomment and change the directory if mutexes are file-based and the default +# mutex file directory is not on a local disk or is not appropriate for some +# other reason. +# +# Mutex default:logs + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +#Listen 12.34.56.78:80 +Listen 80 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +LoadModule mpm_event_module modules/mod_mpm_event.so +#LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +#LoadModule mpm_worker_module modules/mod_mpm_worker.so +LoadModule authn_file_module modules/mod_authn_file.so +#LoadModule authn_dbm_module modules/mod_authn_dbm.so +#LoadModule authn_anon_module modules/mod_authn_anon.so +#LoadModule authn_dbd_module modules/mod_authn_dbd.so +#LoadModule authn_socache_module modules/mod_authn_socache.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authz_groupfile_module modules/mod_authz_groupfile.so +LoadModule authz_user_module modules/mod_authz_user.so +#LoadModule authz_dbm_module modules/mod_authz_dbm.so +#LoadModule authz_owner_module modules/mod_authz_owner.so +#LoadModule authz_dbd_module modules/mod_authz_dbd.so +LoadModule authz_core_module modules/mod_authz_core.so +#LoadModule authnz_ldap_module modules/mod_authnz_ldap.so +#LoadModule authnz_fcgi_module modules/mod_authnz_fcgi.so +LoadModule access_compat_module modules/mod_access_compat.so +LoadModule auth_basic_module modules/mod_auth_basic.so +#LoadModule auth_form_module modules/mod_auth_form.so +#LoadModule auth_digest_module modules/mod_auth_digest.so +#LoadModule allowmethods_module modules/mod_allowmethods.so +#LoadModule isapi_module modules/mod_isapi.so +#LoadModule file_cache_module modules/mod_file_cache.so +#LoadModule cache_module modules/mod_cache.so +#LoadModule cache_disk_module modules/mod_cache_disk.so +#LoadModule cache_socache_module modules/mod_cache_socache.so +#LoadModule socache_shmcb_module modules/mod_socache_shmcb.so +#LoadModule socache_dbm_module modules/mod_socache_dbm.so +#LoadModule socache_memcache_module modules/mod_socache_memcache.so +#LoadModule socache_redis_module modules/mod_socache_redis.so +#LoadModule watchdog_module modules/mod_watchdog.so +#LoadModule macro_module modules/mod_macro.so +#LoadModule dbd_module modules/mod_dbd.so +#LoadModule bucketeer_module modules/mod_bucketeer.so +#LoadModule dumpio_module modules/mod_dumpio.so +#LoadModule echo_module modules/mod_echo.so +#LoadModule example_hooks_module modules/mod_example_hooks.so +#LoadModule case_filter_module modules/mod_case_filter.so +#LoadModule case_filter_in_module modules/mod_case_filter_in.so +#LoadModule example_ipc_module modules/mod_example_ipc.so +#LoadModule buffer_module modules/mod_buffer.so +#LoadModule data_module modules/mod_data.so +#LoadModule ratelimit_module modules/mod_ratelimit.so +LoadModule reqtimeout_module modules/mod_reqtimeout.so +#LoadModule ext_filter_module modules/mod_ext_filter.so +#LoadModule request_module modules/mod_request.so +#LoadModule include_module modules/mod_include.so +LoadModule filter_module modules/mod_filter.so +#LoadModule reflector_module modules/mod_reflector.so +#LoadModule substitute_module modules/mod_substitute.so +#LoadModule sed_module modules/mod_sed.so +#LoadModule charset_lite_module modules/mod_charset_lite.so +#LoadModule deflate_module modules/mod_deflate.so +#LoadModule xml2enc_module modules/mod_xml2enc.so +#LoadModule proxy_html_module modules/mod_proxy_html.so +#LoadModule brotli_module modules/mod_brotli.so +LoadModule mime_module modules/mod_mime.so +#LoadModule ldap_module modules/mod_ldap.so +LoadModule log_config_module modules/mod_log_config.so +#LoadModule log_debug_module modules/mod_log_debug.so +#LoadModule log_forensic_module modules/mod_log_forensic.so +#LoadModule logio_module modules/mod_logio.so +#LoadModule lua_module modules/mod_lua.so +LoadModule env_module modules/mod_env.so +#LoadModule mime_magic_module modules/mod_mime_magic.so +#LoadModule cern_meta_module modules/mod_cern_meta.so +#LoadModule expires_module modules/mod_expires.so +LoadModule headers_module modules/mod_headers.so +#LoadModule ident_module modules/mod_ident.so +#LoadModule usertrack_module modules/mod_usertrack.so +#LoadModule unique_id_module modules/mod_unique_id.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule version_module modules/mod_version.so +#LoadModule remoteip_module modules/mod_remoteip.so +#LoadModule proxy_module modules/mod_proxy.so +#LoadModule proxy_connect_module modules/mod_proxy_connect.so +#LoadModule proxy_ftp_module modules/mod_proxy_ftp.so +#LoadModule proxy_http_module modules/mod_proxy_http.so +#LoadModule proxy_fcgi_module modules/mod_proxy_fcgi.so +#LoadModule proxy_scgi_module modules/mod_proxy_scgi.so +#LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so +#LoadModule proxy_fdpass_module modules/mod_proxy_fdpass.so +#LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so +#LoadModule proxy_ajp_module modules/mod_proxy_ajp.so +#LoadModule proxy_balancer_module modules/mod_proxy_balancer.so +#LoadModule proxy_express_module modules/mod_proxy_express.so +#LoadModule proxy_hcheck_module modules/mod_proxy_hcheck.so +#LoadModule session_module modules/mod_session.so +#LoadModule session_cookie_module modules/mod_session_cookie.so +#LoadModule session_crypto_module modules/mod_session_crypto.so +#LoadModule session_dbd_module modules/mod_session_dbd.so +#LoadModule slotmem_shm_module modules/mod_slotmem_shm.so +#LoadModule slotmem_plain_module modules/mod_slotmem_plain.so +#LoadModule ssl_module modules/mod_ssl.so +#LoadModule optional_hook_export_module modules/mod_optional_hook_export.so +#LoadModule optional_hook_import_module modules/mod_optional_hook_import.so +#LoadModule optional_fn_import_module modules/mod_optional_fn_import.so +#LoadModule optional_fn_export_module modules/mod_optional_fn_export.so +#LoadModule dialup_module modules/mod_dialup.so +#LoadModule http2_module modules/mod_http2.so +#LoadModule proxy_http2_module modules/mod_proxy_http2.so +#LoadModule md_module modules/mod_md.so +#LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so +#LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so +#LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so +#LoadModule lbmethod_heartbeat_module modules/mod_lbmethod_heartbeat.so +LoadModule unixd_module modules/mod_unixd.so +#LoadModule heartbeat_module modules/mod_heartbeat.so +#LoadModule heartmonitor_module modules/mod_heartmonitor.so +#LoadModule dav_module modules/mod_dav.so +LoadModule status_module modules/mod_status.so +LoadModule autoindex_module modules/mod_autoindex.so +#LoadModule asis_module modules/mod_asis.so +#LoadModule info_module modules/mod_info.so +#LoadModule suexec_module modules/mod_suexec.so + + #LoadModule cgid_module modules/mod_cgid.so + + + #LoadModule cgi_module modules/mod_cgi.so + +#LoadModule dav_fs_module modules/mod_dav_fs.so +#LoadModule dav_lock_module modules/mod_dav_lock.so +#LoadModule vhost_alias_module modules/mod_vhost_alias.so +#LoadModule negotiation_module modules/mod_negotiation.so +LoadModule dir_module modules/mod_dir.so +#LoadModule imagemap_module modules/mod_imagemap.so +#LoadModule actions_module modules/mod_actions.so +#LoadModule speling_module modules/mod_speling.so +#LoadModule userdir_module modules/mod_userdir.so +LoadModule alias_module modules/mod_alias.so +#LoadModule rewrite_module modules/mod_rewrite.so + + +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +# +User www-data +Group www-data + + + +# 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# definition. These values also provide defaults for +# any containers you may define later in the file. +# +# All of these directives may appear inside containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin you@example.com + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# +#ServerName www.example.com:80 + +# +# Deny access to the entirety of your server's filesystem. You must +# explicitly permit access to web content directories in other +# blocks below. +# + + AllowOverride none + Require all denied + + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/usr/local/apache2/htdocs" + + # + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + # + Options Indexes FollowSymLinks + + # + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # AllowOverride FileInfo AuthConfig Limit + # + AllowOverride None + + # + # Controls who can get stuff from this server. + # + Require all granted + + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# + + DirectoryIndex index.html + + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog /proc/self/fd/2 + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + + + # + # The following directives define some format nicknames for use with + # a CustomLog directive (see below). + # + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + # + # The location and format of the access logfile (Common Logfile Format). + # If you do not define any access logfiles within a + # container, they will be logged here. Contrariwise, if you *do* + # define per- access logfiles, transactions will be + # logged therein and *not* in this file. + # + CustomLog /proc/self/fd/1 common + + # + # If you prefer a logfile with access, agent, and referer information + # (Combined Logfile Format) you can use the following directive. + # + #CustomLog "logs/access_log" combined + + + + # + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a section to allow access to + # the filesystem path. + + # + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + # + ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/" + + + + + # + # ScriptSock: On threaded servers, designate the path to the UNIX + # socket used to communicate with the CGI daemon of mod_cgid. + # + #Scriptsock cgisock + + +# +# "/usr/local/apache2/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# + + AllowOverride None + Options None + Require all granted + + + + # + # Avoid passing HTTP_PROXY environment to CGI's on this or any proxied + # backend servers which have lingering "httpoxy" defects. + # 'Proxy' request header is undefined by the IETF, not listed by IANA + # + RequestHeader unset Proxy early + + + + # + # TypesConfig points to the file containing the list of mappings from + # filename extension to MIME-type. + # + TypesConfig conf/mime.types + + # + # AddType allows you to add to or override the MIME configuration + # file specified in TypesConfig for specific file types. + # + #AddType application/x-gzip .tgz + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # + #AddEncoding x-compress .Z + #AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + AddType application/x-compress .Z + AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers": + # actions unrelated to filetype. These can be either built into the server + # or added with the Action directive (see below) + # + # To use CGI scripts outside of ScriptAliased directories: + # (You will also need to add "ExecCGI" to the "Options" directive.) + # + #AddHandler cgi-script .cgi + + # For type maps (negotiated resources): + #AddHandler type-map var + + # + # Filters allow you to process content before it is sent to the client. + # + # To parse .shtml files for server-side includes (SSI): + # (You will also need to add "Includes" to the "Options" directive.) + # + #AddType text/html .shtml + #AddOutputFilter INCLUDES .shtml + + +# +# The mod_mime_magic module allows the server to use various hints from the +# contents of the file itself to determine its type. The MIMEMagicFile +# directive tells the module where the hint definitions are located. +# +#MIMEMagicFile conf/magic + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server took a hit and needs to rest." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# MaxRanges: Maximum number of Ranges in a request before +# returning the entire resource, or one of the special +# values 'default', 'none' or 'unlimited'. +# Default setting is to accept 200 Ranges. +#MaxRanges unlimited + +# +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall may be used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +# Defaults: EnableMMAP On, EnableSendfile Off +# +#EnableMMAP off +#EnableSendfile on + +# Supplemental configuration +# +# The configuration files in the conf/extra/ directory can be +# included to add extra features or to modify the default configuration of +# the server, or you may simply copy their contents here and change as +# necessary. + +# Server-pool management (MPM specific) +Include conf/extra/httpd-mpm.conf + +# Multi-language error messages +#Include conf/extra/httpd-multilang-errordoc.conf + +# Fancy directory listings +#Include conf/extra/httpd-autoindex.conf + +# Language settings +#Include conf/extra/httpd-languages.conf + +# User home directories +#Include conf/extra/httpd-userdir.conf + +# Real-time info on requests and configuration +#Include conf/extra/httpd-info.conf + +# Virtual hosts +#Include conf/extra/httpd-vhosts.conf + +# Local access to the Apache HTTP Server Manual +#Include conf/extra/httpd-manual.conf + +# Distributed authoring and versioning (WebDAV) +#Include conf/extra/httpd-dav.conf + +# Various default settings +#Include conf/extra/httpd-default.conf + +# Configure mod_proxy_html to understand HTML4/XHTML1 + +Include conf/extra/proxy-html.conf + + +# Secure (SSL/TLS) connections +#Include conf/extra/httpd-ssl.conf +# +# Note: The following must be present to support +# starting without SSL on platforms with no /dev/random equivalent +# but a statically compiled-in mod_ssl. +# + +SSLRandomSeed startup builtin +SSLRandomSeed connect builtin + + diff --git a/chapters/compute/processes-threads-apache2/guides/apache2/support/make_conn.py b/chapters/compute/processes-threads-apache2/guides/apache2/support/make_conn.py new file mode 100644 index 0000000000..960bb0ea9a --- /dev/null +++ b/chapters/compute/processes-threads-apache2/guides/apache2/support/make_conn.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: BSD-3-Clause + +import socket +import sys + +NUM_CONNS = 100 + +if len(sys.argv) != 3: + print("Usage: make_conn.py host port") + exit() + +host = sys.argv[1] +port = int(sys.argv[2]) + +conns = [] + +for i in range(NUM_CONNS): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + s.connect((host, port)) + + s.send(b"A") + + conns.append(s) + +input("Press ENTER to exit") diff --git a/chapters/compute/processes-threads-apache2/reading/processes-threads-apache2.md b/chapters/compute/processes-threads-apache2/reading/processes-threads-apache2.md new file mode 100644 index 0000000000..acb5321c56 --- /dev/null +++ b/chapters/compute/processes-threads-apache2/reading/processes-threads-apache2.md @@ -0,0 +1,40 @@ +# Usage of Processes and Threads in `apache2` + +We'll take a look at how a real-world application - the `apache2` HTTP server - makes use of processes and threads. +Since the server must be able to handle multiple clients at the same time, it must therefore use some form of concurrency. +When a new client arrives, the server offloads the work of interacting with that client to another process or thread. + +The choice of whether to use multiple processes or threads is not baked into the code. +Instead, `apache2` provides a couple of modules called MPMs (Multi-Processing Modules). +Each module implements a different concurrency model, and the users can pick whatever module best fits their needs by editing the server configuration files. + +The most common MPMs are + +- `prefork`: there are multiple worker processes, each process is single-threaded and handles one client request at a time +- `worker`: there are multiple worker processes, each process is multi-threaded, and each thread handles one client request at a time +- `event`: same as `worker` but designed to better handle some particular use cases + +In principle, `prefork` provides more stability and backwards compatibility, but it has a bigger overhead. +On the other hand, `worker` and `event` are more scalable, and thus able to handle more simultaneous connections, due to the usage of threads. +On modern systems, `event` is almost always the default. + +## Conclusion + +So far, you've probably seen that spawning a process can "use" a different program (hence the path in the args of `system` or `Popen`), but some languages such as Python allow you to spawn a process that executes a function from the same script. +A thread, however, can only start from a certain entry point **within the current address space**, as it is bound to the same process. +Concretely, a process is but a group of threads. +For this reason, when we talk about scheduling or synchronization, we talk about threads. +A thread is, thus, an abstraction of a task running on a CPU core. +A process is a logical group of such tasks. + +We can sum up what we've learned so far by saying that processes are better used for separate, independent work, such as the different connections handled by a server. +Conversely, threads are better suited for replicated work: when the same task has to be performed on multiple cores. +However, replicated work can also be suited for processes. +Distributed applications, however, leverage different processes as this allows them to run on multiple physical machines at once. +This is required by the very large workloads such applications are commonly required to process. + +These rules are not set in stone, though. +Like we saw in the `apache2` example, the server uses multiple threads as well as multiple processes. +This provides a degree of stability - if one worker thread crashes, it will only crash the other threads belonging to the same process - while still taking advantage of the light resource usage inherent to threads. + +These kinds of trade-offs are a normal part of the development of real-world applications. diff --git a/content/chapters/compute/lab/quiz/cause-of-file-not-found-error.md b/chapters/compute/processes/drills/questions/cause-of-file-not-found-error.md similarity index 100% rename from content/chapters/compute/lab/quiz/cause-of-file-not-found-error.md rename to chapters/compute/processes/drills/questions/cause-of-file-not-found-error.md diff --git a/content/chapters/compute/lab/quiz/create-sleepy-process-ending.md b/chapters/compute/processes/drills/questions/create-sleepy-process-ending.md similarity index 91% rename from content/chapters/compute/lab/quiz/create-sleepy-process-ending.md rename to chapters/compute/processes/drills/questions/create-sleepy-process-ending.md index a4961ddc18..50c1616603 100644 --- a/content/chapters/compute/lab/quiz/create-sleepy-process-ending.md +++ b/chapters/compute/processes/drills/questions/create-sleepy-process-ending.md @@ -7,7 +7,7 @@ Use [`system`'s man page](https://man7.org/linux/man-pages/man3/system.3.html) t ## Question Answers -- Because the code is unoptimised (the default optimisation level is `-O0`) +- Because the code is unoptimized (the default optimisation level is `-O0`) - Because the operating system takes very long to finish the process diff --git a/content/chapters/compute/lecture/quiz/exec-without-fork.md b/chapters/compute/processes/drills/questions/exec-without-fork.md similarity index 93% rename from content/chapters/compute/lecture/quiz/exec-without-fork.md rename to chapters/compute/processes/drills/questions/exec-without-fork.md index fa87312549..563c144111 100644 --- a/content/chapters/compute/lecture/quiz/exec-without-fork.md +++ b/chapters/compute/processes/drills/questions/exec-without-fork.md @@ -26,4 +26,4 @@ Why does the shell **NOT** simply call `exec("/bin/ls")`? If the shell simply `exec`s `"/bin/ls"`, the `bash` process is entirely replaced by the `ls` process. Therefore, when the `ls` process ends, the old `bash` process is no more and the terminal closes. -This way, each command would close the termianl. +This way, each command would close the terminal. diff --git a/content/chapters/compute/lab/quiz/mini-shell-stops-after-command.md b/chapters/compute/processes/drills/questions/mini-shell-stops-after-command.md similarity index 100% rename from content/chapters/compute/lab/quiz/mini-shell-stops-after-command.md rename to chapters/compute/processes/drills/questions/mini-shell-stops-after-command.md diff --git a/content/chapters/compute/lab/quiz/parent-of-sleep-processes.md b/chapters/compute/processes/drills/questions/parent-of-sleep-processes.md similarity index 100% rename from content/chapters/compute/lab/quiz/parent-of-sleep-processes.md rename to chapters/compute/processes/drills/questions/parent-of-sleep-processes.md diff --git a/content/chapters/compute/lecture/quiz/process-creation.md b/chapters/compute/processes/drills/questions/process-creation.md similarity index 100% rename from content/chapters/compute/lecture/quiz/process-creation.md rename to chapters/compute/processes/drills/questions/process-creation.md diff --git a/content/chapters/compute/lab/quiz/processes-speedup.md b/chapters/compute/processes/drills/questions/processes-speedup.md similarity index 100% rename from content/chapters/compute/lab/quiz/processes-speedup.md rename to chapters/compute/processes/drills/questions/processes-speedup.md diff --git a/content/chapters/compute/lab/quiz/who-calls-execve-parent.md b/chapters/compute/processes/drills/questions/who-calls-execve-parent.md similarity index 100% rename from content/chapters/compute/lab/quiz/who-calls-execve-parent.md rename to chapters/compute/processes/drills/questions/who-calls-execve-parent.md diff --git a/chapters/compute/processes/drills/tasks/create-process/README.md b/chapters/compute/processes/drills/tasks/create-process/README.md new file mode 100644 index 0000000000..6ef2a22719 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/README.md @@ -0,0 +1,11 @@ +# Create Process + +Enter the `create-process/support/` folder and go through the practice items below. + +1. Change the return value of the child process so that the value displayed by the parent is changed. + +1. Create a child process of the newly created child. + +Use a similar logic and a similar set of prints to those in the support code. +Take a look at the printed PIDs. +Make sure the PPID of the "grandchild" is the PID of the child, whose PPID is, in turn, the PID of the parent. diff --git a/content/chapters/compute/lab/solution/create-process/.gitignore b/chapters/compute/processes/drills/tasks/create-process/solution/.gitignore similarity index 100% rename from content/chapters/compute/lab/solution/create-process/.gitignore rename to chapters/compute/processes/drills/tasks/create-process/solution/.gitignore diff --git a/chapters/compute/processes/drills/tasks/create-process/solution/Makefile b/chapters/compute/processes/drills/tasks/create-process/solution/Makefile new file mode 100644 index 0000000000..5a77f334c3 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/solution/Makefile @@ -0,0 +1,41 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +SRCS = fork.c +OBJS = $(SRCS:.c=.o) +BINARIES = fork + +# Default rule: Build everything +all: $(BINARIES) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile object files from source files +%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create fork binary +fork: fork.o $(LOGGER) + $(CC) $(CFLAGS) fork.o $(LOGGER) -o fork $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/create-process/fork.c b/chapters/compute/processes/drills/tasks/create-process/solution/fork.c similarity index 97% rename from content/chapters/compute/lab/solution/create-process/fork.c rename to chapters/compute/processes/drills/tasks/create-process/solution/fork.c index 66385631c1..6c7881a546 100644 --- a/content/chapters/compute/lab/solution/create-process/fork.c +++ b/chapters/compute/processes/drills/tasks/create-process/solution/fork.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/compute/processes/drills/tasks/create-process/solution/utils/log/CPPLINT.cfg b/chapters/compute/processes/drills/tasks/create-process/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/processes/drills/tasks/create-process/solution/utils/log/log.c b/chapters/compute/processes/drills/tasks/create-process/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/processes/drills/tasks/create-process/solution/utils/log/log.h b/chapters/compute/processes/drills/tasks/create-process/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/processes/drills/tasks/create-process/solution/utils/sock/sock_util.c b/chapters/compute/processes/drills/tasks/create-process/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/processes/drills/tasks/create-process/solution/utils/sock/sock_util.h b/chapters/compute/processes/drills/tasks/create-process/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/processes/drills/tasks/create-process/solution/utils/utils.h b/chapters/compute/processes/drills/tasks/create-process/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/support/create-process/.gitignore b/chapters/compute/processes/drills/tasks/create-process/support/.gitignore similarity index 100% rename from content/chapters/compute/lab/support/create-process/.gitignore rename to chapters/compute/processes/drills/tasks/create-process/support/.gitignore diff --git a/chapters/compute/processes/drills/tasks/create-process/support/Makefile b/chapters/compute/processes/drills/tasks/create-process/support/Makefile new file mode 100644 index 0000000000..5a77f334c3 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/support/Makefile @@ -0,0 +1,41 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +SRCS = fork.c +OBJS = $(SRCS:.c=.o) +BINARIES = fork + +# Default rule: Build everything +all: $(BINARIES) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile object files from source files +%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create fork binary +fork: fork.o $(LOGGER) + $(CC) $(CFLAGS) fork.o $(LOGGER) -o fork $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean \ No newline at end of file diff --git a/content/chapters/compute/lab/support/create-process/fork.c b/chapters/compute/processes/drills/tasks/create-process/support/fork.c similarity index 95% rename from content/chapters/compute/lab/support/create-process/fork.c rename to chapters/compute/processes/drills/tasks/create-process/support/fork.c index 41892a646c..1013fd5639 100644 --- a/content/chapters/compute/lab/support/create-process/fork.c +++ b/chapters/compute/processes/drills/tasks/create-process/support/fork.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/compute/processes/drills/tasks/create-process/support/utils/log/CPPLINT.cfg b/chapters/compute/processes/drills/tasks/create-process/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/processes/drills/tasks/create-process/support/utils/log/log.c b/chapters/compute/processes/drills/tasks/create-process/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/processes/drills/tasks/create-process/support/utils/log/log.h b/chapters/compute/processes/drills/tasks/create-process/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/processes/drills/tasks/create-process/support/utils/sock/sock_util.c b/chapters/compute/processes/drills/tasks/create-process/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/processes/drills/tasks/create-process/support/utils/sock/sock_util.h b/chapters/compute/processes/drills/tasks/create-process/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/processes/drills/tasks/create-process/support/utils/utils.h b/chapters/compute/processes/drills/tasks/create-process/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/processes/drills/tasks/create-process/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/processes/drills/tasks/mini-shell/.gitignore b/chapters/compute/processes/drills/tasks/mini-shell/.gitignore new file mode 100644 index 0000000000..1715080a59 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/.gitignore @@ -0,0 +1,3 @@ +support/ +mini-shell +*.o diff --git a/chapters/compute/processes/drills/tasks/mini-shell/Makefile b/chapters/compute/processes/drills/tasks/mini-shell/Makefile new file mode 100644 index 0000000000..12c6b032d2 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/Makefile @@ -0,0 +1,8 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + $(PYTHON) $(SCRIPT) --input ./solution/ --output ./support/ + +clean: + rm -rf support \ No newline at end of file diff --git a/chapters/compute/processes/drills/tasks/mini-shell/README.md b/chapters/compute/processes/drills/tasks/mini-shell/README.md new file mode 100644 index 0000000000..2d8d414d57 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/README.md @@ -0,0 +1,18 @@ +# Mini-shell + +As you might remember, to create a new process you need to use `fork` (or `clone`) and `exec` system calls. +If you don't take a look at [what happens under the hood when you use `system`](guides/system-dissected.md). + +1. With this knowledge in mind, let's implement our own mini-shell. + Start from the skeleton code in `support/mini-shell/mini_shell.c`. + We're already running our Bash interpreter from the command-line, so there's no need to `exec` another Bash from it. + Simply `exec` the command. + + [Quiz](../drills/questions/mini-shell-stops-after-command.md) + + So we need a way to "save" the `mini_shell` process before `exec()`-ing our command. + Find a way to do this. + + > **Hint**: You can see what `sleepy` does and draw inspiration from there. + > Use `strace` to also list the calls to `clone()` performed by `sleepy` or its children. + > [Remember](./guides/clone.md) what `clone()` is used for and use its parameters to deduce which of the two scenarios happens to `sleepy`. diff --git a/chapters/compute/processes/drills/tasks/mini-shell/generate_skels.py b/chapters/compute/processes/drills/tasks/mini-shell/generate_skels.py new file mode 100644 index 0000000000..2589f9b05e --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/generate_skels.py @@ -0,0 +1,150 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/chapters/compute/processes/drills/tasks/mini-shell/solution/.gitignore b/chapters/compute/processes/drills/tasks/mini-shell/solution/.gitignore new file mode 100644 index 0000000000..b5fa44dea9 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/solution/.gitignore @@ -0,0 +1 @@ +mini-shell diff --git a/chapters/compute/processes/drills/tasks/mini-shell/solution/Makefile b/chapters/compute/processes/drills/tasks/mini-shell/solution/Makefile new file mode 100644 index 0000000000..ae810113b5 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/solution/Makefile @@ -0,0 +1,41 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +SRCS = mini_shell.c +OBJS = $(SRCS:.c=.o) +BINARIES = mini_shell + +# Default rule: Build everything +all: $(BINARIES) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile object files from source files +%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create mini_shell binary +mini_shell: mini_shell.o $(LOGGER) + $(CC) $(CFLAGS) mini_shell.o $(LOGGER) -o mini_shell $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/mini-shell/mini_shell.c b/chapters/compute/processes/drills/tasks/mini-shell/solution/mini_shell.c similarity index 91% rename from content/chapters/compute/lab/solution/mini-shell/mini_shell.c rename to chapters/compute/processes/drills/tasks/mini-shell/solution/mini_shell.c index 6cd7914e9a..3eb928f916 100644 --- a/content/chapters/compute/lab/solution/mini-shell/mini_shell.c +++ b/chapters/compute/processes/drills/tasks/mini-shell/solution/mini_shell.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include @@ -39,7 +39,7 @@ static int parse_line(char *line) int idx = 0; char *token; char *delim = "=\n";; - char *saveptr; + char *saveptr = NULL; stdin_file = NULL; stdout_file = NULL; @@ -79,14 +79,14 @@ static int parse_line(char *line) */ static void simple_cmd(char **args) { + /* REPLACE 1 */ + /* (void) args; */ + + /* TODO 27: Create a process to execute the command, using `execvp` to launch the new process. */ pid_t pid; pid_t wait_ret; int status; - /** - * TODO - Create a process to execute the command. - * Use `execvp` to launch the new process. - */ pid = fork(); switch (pid) { case -1: diff --git a/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/log/CPPLINT.cfg b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/log/log.c b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/log/log.h b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/sock/sock_util.c b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/sock/sock_util.h b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/utils.h b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/processes/drills/tasks/mini-shell/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/processes/drills/tasks/sleepy/README.md b/chapters/compute/processes/drills/tasks/sleepy/README.md new file mode 100644 index 0000000000..29bed6b91a --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/README.md @@ -0,0 +1,97 @@ +# Creating a process + +## Higher level - Python + +Enter the `sleepy/support/` folder and go through the practice items below. + +Head over to `sleepy_creator.py`. + +1. Solve `TODO 1`: use `subprocess.Popen()` to spawn 10 `sleep 1000` processes. + + Start the script: + + ```console + student@os:~/.../tasks/sleepy/support$ python3 sleepy_creator.py + ``` + + Look for the parent process: + + ```console + student@os:~$ ps -e -H -o pid,ppid,cmd | (head -1; grep "python3 sleepy_creator.py") + ``` + + It is a `python3` process, as this is the interpreter that runs the script, but we call it the `sleepy_creator.py` process for simplicity. + No output will be provided by the above command, as the parent process (`sleepy_creator.py`) dies before its child processes (the 10 `sleep 1000` subprocesses) finish their execution. + The parent process of the newly created child processes is an `init`-like process: either `systemd`/`init` or another system process that adopts orphan processes. + Look for the `sleep` child processes using: + + ```console + student@os:~$ ps -e -H -o pid,ppid,cmd | (head -1; grep sleep) + PID PPID CMD + 4164 1680 sleep 1000 + 4165 1680 sleep 1000 + 4166 1680 sleep 1000 + 4167 1680 sleep 1000 + 4168 1680 sleep 1000 + 4169 1680 sleep 1000 + 4170 1680 sleep 1000 + 4171 1680 sleep 1000 + 4172 1680 sleep 1000 + 4173 1680 sleep 1000 + ``` + + Notice that the child processes do not have `sleepy_creator.py` as a parent. + What's more, as you saw above, `sleepy_creator.py` doesn't even exist anymore. + The child processes have been adopted by an `init`-like process (in the output above, that process has PID `1680` - `PPID` stands for _parent process ID_). + + [Quiz](../../questions/parent-of-sleep-processes.md) + +1. Solve `TODO 2`: change the code in `sleepy_creator.py` so that the `sleep 1000` processes remain the children of `sleepy_creator.py`. + This means that the parent / creator process must **not** exit until its children have finished their execution. + In other words, the parent / creator process must **wait** for the termination of its children. + Check out [`Popen.wait()`](https://docs.python.org/3/library/subprocess.html#subprocess.Popen.wait) and add the code that makes the parent / creator process wait for its children. + Before anything, terminate the `sleep` processes created above: + + ```console + student@os:~$ pkill sleep + ``` + + Start the program, again, as you did before: + + ```console + student@os:~/.../tasks/sleepy/support$ python3 sleepy_creator.py + ``` + + On another terminal, verify that `sleepy_creator.py` remains the parent of the `sleep` processes it creates: + + ```console + student@os:~$ ps -e -H -o pid,ppid,cmd | (head -1; grep sleep) + PID PPID CMD + 16107 9855 python3 sleepy_creator.py + 16108 16107 sleep 1000 + 16109 16107 sleep 1000 + 16110 16107 sleep 1000 + 16111 16107 sleep 1000 + 16112 16107 sleep 1000 + 16113 16107 sleep 1000 + 16114 16107 sleep 1000 + 16115 16107 sleep 1000 + 16116 16107 sleep 1000 + 16117 16107 sleep 1000 + ``` + + Note that the parent process `sleepy_creator.py` (`PID 16107`) is still alive, and its child processes (the 10 `sleep 1000`) have its ID as their `PPID`. + You've successfully waited for the child processes to finish their execution. + +## Lower level - C + +Now let's see how to create a child process in C. +There are multiple ways of doing this. +For now, we'll start with a higher-level approach. + +Go to `sleepy/support/sleepy_creator.c` and use [`system`](https://man7.org/linux/man-pages/man3/system.3.html) to create a `sleep 1000` process. + +[Quiz](../../questions/create-sleepy-process-ending.md) + +The `man` page also mentions that `system` calls `fork()` and `exec()` to run the command it's given. +If you want to find out more about them, head over to the [Arena and create your own mini-shell](mini-shell). diff --git a/content/chapters/compute/lab/solution/sleepy/.gitignore b/chapters/compute/processes/drills/tasks/sleepy/solution/.gitignore similarity index 100% rename from content/chapters/compute/lab/solution/sleepy/.gitignore rename to chapters/compute/processes/drills/tasks/sleepy/solution/.gitignore diff --git a/chapters/compute/processes/drills/tasks/sleepy/solution/Makefile b/chapters/compute/processes/drills/tasks/sleepy/solution/Makefile new file mode 120000 index 0000000000..a39be4f20d --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/solution/Makefile @@ -0,0 +1 @@ +../support/Makefile \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/sleepy/sleepy_creator.c b/chapters/compute/processes/drills/tasks/sleepy/solution/sleepy_creator.c similarity index 81% rename from content/chapters/compute/lab/solution/sleepy/sleepy_creator.c rename to chapters/compute/processes/drills/tasks/sleepy/solution/sleepy_creator.c index 2fe32c19c9..6372b83d6b 100644 --- a/content/chapters/compute/lab/solution/sleepy/sleepy_creator.c +++ b/chapters/compute/processes/drills/tasks/sleepy/solution/sleepy_creator.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/compute/lab/solution/sleepy/sleepy_creator.py b/chapters/compute/processes/drills/tasks/sleepy/solution/sleepy_creator.py similarity index 100% rename from content/chapters/compute/lab/solution/sleepy/sleepy_creator.py rename to chapters/compute/processes/drills/tasks/sleepy/solution/sleepy_creator.py diff --git a/chapters/compute/processes/drills/tasks/sleepy/solution/utils/log/CPPLINT.cfg b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/processes/drills/tasks/sleepy/solution/utils/log/log.c b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/processes/drills/tasks/sleepy/solution/utils/log/log.h b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/processes/drills/tasks/sleepy/solution/utils/sock/sock_util.c b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/processes/drills/tasks/sleepy/solution/utils/sock/sock_util.h b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/processes/drills/tasks/sleepy/solution/utils/utils.h b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/support/sleepy/.gitignore b/chapters/compute/processes/drills/tasks/sleepy/support/.gitignore similarity index 100% rename from content/chapters/compute/lab/support/sleepy/.gitignore rename to chapters/compute/processes/drills/tasks/sleepy/support/.gitignore diff --git a/chapters/compute/processes/drills/tasks/sleepy/support/Makefile b/chapters/compute/processes/drills/tasks/sleepy/support/Makefile new file mode 100644 index 0000000000..77ca1648a4 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/support/Makefile @@ -0,0 +1,41 @@ +BINARY = sleepy_creator + +all: $(BINARY) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARY): $(OBJS) $(LOGGER) + $(CC) $^ $(LDFLAGS) -o $@ $(LDLIBS) + +clean:: + -rm -f $(BINARY) + +.PHONY: all clean + +clean:: + -rm -f syscalls* diff --git a/content/chapters/compute/lab/support/sleepy/sleepy_creator.c b/chapters/compute/processes/drills/tasks/sleepy/support/sleepy_creator.c similarity index 79% rename from content/chapters/compute/lab/support/sleepy/sleepy_creator.c rename to chapters/compute/processes/drills/tasks/sleepy/support/sleepy_creator.c index d6b9888fef..adaa723b7b 100644 --- a/content/chapters/compute/lab/support/sleepy/sleepy_creator.c +++ b/chapters/compute/processes/drills/tasks/sleepy/support/sleepy_creator.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/compute/lab/support/sleepy/sleepy_creator.py b/chapters/compute/processes/drills/tasks/sleepy/support/sleepy_creator.py similarity index 100% rename from content/chapters/compute/lab/support/sleepy/sleepy_creator.py rename to chapters/compute/processes/drills/tasks/sleepy/support/sleepy_creator.py diff --git a/chapters/compute/processes/drills/tasks/sleepy/support/utils/log/CPPLINT.cfg b/chapters/compute/processes/drills/tasks/sleepy/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/processes/drills/tasks/sleepy/support/utils/log/log.c b/chapters/compute/processes/drills/tasks/sleepy/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/processes/drills/tasks/sleepy/support/utils/log/log.h b/chapters/compute/processes/drills/tasks/sleepy/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/processes/drills/tasks/sleepy/support/utils/sock/sock_util.c b/chapters/compute/processes/drills/tasks/sleepy/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/processes/drills/tasks/sleepy/support/utils/sock/sock_util.h b/chapters/compute/processes/drills/tasks/sleepy/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/processes/drills/tasks/sleepy/support/utils/utils.h b/chapters/compute/processes/drills/tasks/sleepy/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/processes/drills/tasks/sleepy/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/processes/drills/tasks/wait-for-me-processes/README.md b/chapters/compute/processes/drills/tasks/wait-for-me-processes/README.md new file mode 100644 index 0000000000..38b9dcaa68 --- /dev/null +++ b/chapters/compute/processes/drills/tasks/wait-for-me-processes/README.md @@ -0,0 +1,22 @@ +# Wait for Me + +Enter the `wait-for-me/support/` folder and go through the practice items below. + +1. Run the code in `wait_for_me_processes.py`. + The parent process creates one child that writes and message to the given file. + Then the parent reads that message. + Simple enough, right? + But running the code raises a `FileNotFoundError`. + If you inspect the file you gave the script as an argument, it does contain a string. + What's going on? + + [Quiz](../../questions/cause-of-file-not-found-error.md) + + In order to solve race conditions, we need **synchronization**. + This is a mechanism similar to a set of traffic lights in a crossroads. + Just like traffic lights allow some cars to pass only after others have already passed, synchronization is a means for threads to communicate with each other and tell each other to access a resource or not. + + The most basic form of synchronization is **waiting**. + Concretely, if the parent process **waits** for the child to end, we are sure the file is created and its contents are written. + +1. Use `join()` to make the parent wait for its child before reading the file. diff --git a/content/chapters/compute/lab/solution/wait-for-me/wait_for_me_processes.py b/chapters/compute/processes/drills/tasks/wait-for-me-processes/solution/wait_for_me_processes.py similarity index 100% rename from content/chapters/compute/lab/solution/wait-for-me/wait_for_me_processes.py rename to chapters/compute/processes/drills/tasks/wait-for-me-processes/solution/wait_for_me_processes.py diff --git a/content/chapters/compute/lab/support/wait-for-me/wait_for_me_processes.py b/chapters/compute/processes/drills/tasks/wait-for-me-processes/support/wait_for_me_processes.py similarity index 100% rename from content/chapters/compute/lab/support/wait-for-me/wait_for_me_processes.py rename to chapters/compute/processes/drills/tasks/wait-for-me-processes/support/wait_for_me_processes.py diff --git a/chapters/compute/processes/guides/create-process/README.md b/chapters/compute/processes/guides/create-process/README.md new file mode 100644 index 0000000000..bce390ae34 --- /dev/null +++ b/chapters/compute/processes/guides/create-process/README.md @@ -0,0 +1,14 @@ +# Baby steps - Python + +Run the code in `create-process/support/popen.py`. +It simply spawns a new process running the `ls` command using [`subprocess.Popen()`](https://docs.python.org/3/library/subprocess.html#subprocess.Popen). +Do not worry about the huge list of arguments that `Popen()` takes. +They are used for **inter-process-communication**. +You'll learn more about this in the [Application Interaction chapter]. + +Note that this usage of `Popen()` is not entirely correct. +You'll discover why in the next exercise, but for now focus on simply understanding how to use `Popen()` on its own. + +Now change the command to anything you want. +Also give it some arguments. +From the outside, it's as if you were running these commands from the terminal. diff --git a/content/chapters/compute/lab/support/create-process/popen.py b/chapters/compute/processes/guides/create-process/support/popen.py similarity index 100% rename from content/chapters/compute/lab/support/create-process/popen.py rename to chapters/compute/processes/guides/create-process/support/popen.py diff --git a/chapters/compute/processes/guides/sum-array-processes/README.md b/chapters/compute/processes/guides/sum-array-processes/README.md new file mode 100644 index 0000000000..c3f5721b8f --- /dev/null +++ b/chapters/compute/processes/guides/sum-array-processes/README.md @@ -0,0 +1,54 @@ +# Sum Array Processes + +## Sum of the Elements in an Array + +Let's assume we only have one process on our system, and that process knows how to add the numbers in an array. +It can use however many resources it wants, since there is no other process to contest it. +It would probably look like the code in `sum-array/support/c/sum_array_sequential.c`. +The program also measures the time spent computing the sum. +Let's compile and run it: + +```console +student@os:~/.../sum-array/support/c$ ./sum_array_sequential +Array sum is: 49945994146 +Time spent: 127 ms +``` + +You will most likely get a different sum (because the array is made up of random numbers) and a different time than the ones shown above. +This is perfectly fine. +Use these examples qualitatively, not quantitatively. + +## Spreading the Work Among Other Processes + +Due to how it's implemented so far, our program only uses one of our CPU's cores. +We never tell it to distribute its workload to other cores. +This is wasteful as the rest of our cores remain unused: + +```console +student@os:~$ lscpu | grep ^CPU\(s\): +CPU(s): 8 +``` + +We have 7 more cores waiting to add numbers in our array. + +![What if we used 100% of the CPU?](../../media/100-percent-cpu.jpeg) + +What if we use 7 more processes and spread the task of adding the numbers in this array between them? +If we split the array into several equal parts and designate a separate process to calculate the sum of each part, we should get a speedup because now the work performed by each individual process is reduced. + +Let's take it methodically. +Compile and run `sum_array_processes.c` using 1, 2, 4 and 8 processes respectively. +If your system only has 4 cores ([hyperthreading](https://www.intel.com/content/www/us/en/gaming/resources/hyper-threading.html) included), limit your runs to 4 processes. +Note the running times for each number of processes. +We expect the speedups compared to our reference run to be 1, 2, 4 and 8 respectively, right? + +[Quiz](../../drills/questions/processes-speedup.md) + +You most likely did get some speedup, especially when using 8 processes. +Now we will try to improve this speedup by using **threads** instead. + +Also notice that we're not using hundreds or thousands of processes. +Assuming our system has 8 cores, only 8 _threads_ (we'll see this later in the lab) can run at the same time. +In general, **the maximum number of threads that can run at the same time is equal to the number of cores**. +In our example, each process only has one thread: its main thread. +So by consequence and by forcing the terminology (because it's the main thread of these processes that is running, not the processes themselves), we can only run in parallel a number of processes equal to at most the number of cores. diff --git a/content/chapters/compute/lab/support/sum-array/c/.gitignore b/chapters/compute/processes/guides/sum-array-processes/support/c/.gitignore similarity index 100% rename from content/chapters/compute/lab/support/sum-array/c/.gitignore rename to chapters/compute/processes/guides/sum-array-processes/support/c/.gitignore diff --git a/chapters/compute/processes/guides/sum-array-processes/support/c/Makefile b/chapters/compute/processes/guides/sum-array-processes/support/c/Makefile new file mode 100644 index 0000000000..55db62eb03 --- /dev/null +++ b/chapters/compute/processes/guides/sum-array-processes/support/c/Makefile @@ -0,0 +1,47 @@ +BINARIES = sum_array_sequential sum_array_threads sum_array_processes sum_array_threads_openmp +CFLAGS += -fopenmp +LDFLAGS += -fopenmp + +all: $(BINARIES) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARIES): $(LOGGER) + +clean:: + -rm -f $(BINARIES) + +.PHONY: all clean + +sum_array_sequential: sum_array_sequential.o generate_random_array.o + +sum_array_threads: sum_array_threads.o generate_random_array.o + +sum_array_processes: sum_array_processes.o generate_random_array.o + +sum_array_threads_openmp: sum_array_threads_openmp.o generate_random_array.o diff --git a/content/chapters/compute/lab/support/sum-array-bugs/seg-fault/generate_random_array.c b/chapters/compute/processes/guides/sum-array-processes/support/c/generate_random_array.c similarity index 100% rename from content/chapters/compute/lab/support/sum-array-bugs/seg-fault/generate_random_array.c rename to chapters/compute/processes/guides/sum-array-processes/support/c/generate_random_array.c diff --git a/content/chapters/compute/lab/support/sum-array/c/include/array_utils.h b/chapters/compute/processes/guides/sum-array-processes/support/c/include/array_utils.h similarity index 100% rename from content/chapters/compute/lab/support/sum-array/c/include/array_utils.h rename to chapters/compute/processes/guides/sum-array-processes/support/c/include/array_utils.h diff --git a/content/chapters/compute/lab/support/sum-array-bugs/seg-fault/include/generate_random_array.h b/chapters/compute/processes/guides/sum-array-processes/support/c/include/generate_random_array.h similarity index 100% rename from content/chapters/compute/lab/support/sum-array-bugs/seg-fault/include/generate_random_array.h rename to chapters/compute/processes/guides/sum-array-processes/support/c/include/generate_random_array.h diff --git a/content/chapters/compute/lab/support/sum-array/c/sum_array_processes.c b/chapters/compute/processes/guides/sum-array-processes/support/c/sum_array_processes.c similarity index 100% rename from content/chapters/compute/lab/support/sum-array/c/sum_array_processes.c rename to chapters/compute/processes/guides/sum-array-processes/support/c/sum_array_processes.c diff --git a/content/chapters/compute/lab/support/sum-array/c/sum_array_sequential.c b/chapters/compute/processes/guides/sum-array-processes/support/c/sum_array_sequential.c similarity index 100% rename from content/chapters/compute/lab/support/sum-array/c/sum_array_sequential.c rename to chapters/compute/processes/guides/sum-array-processes/support/c/sum_array_sequential.c diff --git a/content/chapters/compute/lab/support/sum-array/c/sum_array_threads.c b/chapters/compute/processes/guides/sum-array-processes/support/c/sum_array_threads.c similarity index 100% rename from content/chapters/compute/lab/support/sum-array/c/sum_array_threads.c rename to chapters/compute/processes/guides/sum-array-processes/support/c/sum_array_threads.c diff --git a/content/chapters/compute/lab/support/sum-array/c/sum_array_threads_openmp.c b/chapters/compute/processes/guides/sum-array-processes/support/c/sum_array_threads_openmp.c similarity index 100% rename from content/chapters/compute/lab/support/sum-array/c/sum_array_threads_openmp.c rename to chapters/compute/processes/guides/sum-array-processes/support/c/sum_array_threads_openmp.c diff --git a/chapters/compute/processes/guides/sum-array-processes/support/c/utils/log/CPPLINT.cfg b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/processes/guides/sum-array-processes/support/c/utils/log/log.c b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/processes/guides/sum-array-processes/support/c/utils/log/log.h b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.c b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.h b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/processes/guides/sum-array-processes/support/c/utils/utils.h b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/processes/guides/system-dissected/README.md b/chapters/compute/processes/guides/system-dissected/README.md new file mode 100644 index 0000000000..388b5dfeb0 --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/README.md @@ -0,0 +1,40 @@ +# `system` Dissected + +You already know that `system` calls `fork()` and `execve()` to create the new process. +Let's see how and why. +First, we run the following command to trace the `execve()` syscalls used by `sleepy_creator`. +We'll leave `fork()` for later. + +```console +student@os:~/.../sleepy/support$ strace -e execve -ff -o syscalls ./sleepy_creator +``` + +At this point, you will get two files whose names start with `syscalls`, followed by some numbers. +Those numbers are the PIDs of the parent and the child process. +Therefore, the file with the higher number contains logs of the `execve` and `clone` syscalls issued by the parent process, while +the other logs those two syscalls when made by the child process. +Let's take a look at them. +The numbers below will differ from those on your system: + +```console +student@os:~/.../sleepy/support:$ cat syscalls.2523393 # syscalls from parent process +execve("sleepy_creator", ["sleepy_creator"], 0x7ffd2c157758 /* 39 vars */) = 0 +--- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=2523394, si_uid=1052093, si_status=0, si_utime=0, si_stime=0} --- ++++ exited with 0 +++ + +student@os:~/.../sleepy/support:$ cat syscalls.2523394 # syscalls from child process +execve("/bin/sh", ["sh", "-c", "sleep 10"], 0x7ffd36253be8 /* 39 vars */) = 0 +execve("/usr/bin/sleep", ["sleep", "10"], 0x560f41659d40 /* 38 vars */) = 0 ++++ exited with 0 +++ +``` + +[Quiz](../drills/questions/who-calls-execve-parent.md) + +Now notice that the child process doesn't simply call `execve("/usr/bin/sleep" ...)`. +It first changes its virtual address space (VAS) to that of a `bash` process (`execve("/bin/sh" ...)`) and then that `bash` process switches its VAS to `sleep`. +Therefore, calling `system()` is equivalent to running `` in the command-line. + +**Moral of the story**: When spawning a new command, the call order is: + +- parent: `fork()`, `exec()`, `wait()` +- child: `exit()` diff --git a/chapters/compute/processes/guides/system-dissected/support/.gitignore b/chapters/compute/processes/guides/system-dissected/support/.gitignore new file mode 100644 index 0000000000..c2eac32574 --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/support/.gitignore @@ -0,0 +1,2 @@ +sleepy_creator +syscalls.* diff --git a/chapters/compute/processes/guides/system-dissected/support/Makefile b/chapters/compute/processes/guides/system-dissected/support/Makefile new file mode 100644 index 0000000000..48441ba8f6 --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/support/Makefile @@ -0,0 +1,41 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +SRCS = sleepy_creator.c +OBJS = $(SRCS:.c=.o) +BINARIES = sleepy_creator + +# Default rule: Build everything +all: $(BINARIES) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile object files from source files +%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create sleepy_creator binary +sleepy_creator: sleepy_creator.o $(LOGGER) + $(CC) $(CFLAGS) sleepy_creator.o $(LOGGER) -o sleepy_creator $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean \ No newline at end of file diff --git a/chapters/compute/processes/guides/system-dissected/support/sleepy_creator.c b/chapters/compute/processes/guides/system-dissected/support/sleepy_creator.c new file mode 100644 index 0000000000..6372b83d6b --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/support/sleepy_creator.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + +int main(void) +{ + /* TODO 1: Create one `sleep 1000` process using `system`. */ + system("sleep 1000"); + puts("Parent process ending."); + return 0; +} diff --git a/chapters/compute/processes/guides/system-dissected/support/sleepy_creator.py b/chapters/compute/processes/guides/system-dissected/support/sleepy_creator.py new file mode 100644 index 0000000000..256626fc70 --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/support/sleepy_creator.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: BSD-3-Clause + +import subprocess +from sys import exit + +NUM_SLEEPS = 10 + + +def main(): + # TODO 1: Create 10 `sleep 1000` processes using `subprocess.Popen` + # Use the documentation: https://docs.python.org/3/library/subprocess.html#subprocess.Popen + procs = [] + for _ in range(NUM_SLEEPS): + # Create new process and add it to the list of processes. + p = subprocess.Popen(["sleep", "1000"]) + procs.append(p) + + # TODO 2: Make the current process wait for its child processes. + for p in procs: + p.wait() + + +if __name__ == "__main__": + exit(main()) diff --git a/chapters/compute/processes/guides/system-dissected/support/utils/log/CPPLINT.cfg b/chapters/compute/processes/guides/system-dissected/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/processes/guides/system-dissected/support/utils/log/log.c b/chapters/compute/processes/guides/system-dissected/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/processes/guides/system-dissected/support/utils/log/log.h b/chapters/compute/processes/guides/system-dissected/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.c b/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.h b/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/processes/guides/system-dissected/support/utils/utils.h b/chapters/compute/processes/guides/system-dissected/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/processes/guides/system-dissected/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/media/100-percent-cpu.jpeg b/chapters/compute/processes/media/100-percent-cpu.jpeg similarity index 100% rename from content/chapters/compute/lab/media/100-percent-cpu.jpeg rename to chapters/compute/processes/media/100-percent-cpu.jpeg diff --git a/content/chapters/compute/lecture/media/exec.svg b/chapters/compute/processes/media/exec.svg similarity index 100% rename from content/chapters/compute/lecture/media/exec.svg rename to chapters/compute/processes/media/exec.svg diff --git a/content/chapters/compute/lecture/media/fork-exec.svg b/chapters/compute/processes/media/fork-exec.svg similarity index 100% rename from content/chapters/compute/lecture/media/fork-exec.svg rename to chapters/compute/processes/media/fork-exec.svg diff --git a/content/chapters/compute/lecture/media/fork-thread.svg b/chapters/compute/processes/media/fork-thread.svg similarity index 100% rename from content/chapters/compute/lecture/media/fork-thread.svg rename to chapters/compute/processes/media/fork-thread.svg diff --git a/content/chapters/compute/lab/media/loading-of-ls-process.svg b/chapters/compute/processes/media/loading-of-ls-process.svg similarity index 100% rename from content/chapters/compute/lab/media/loading-of-ls-process.svg rename to chapters/compute/processes/media/loading-of-ls-process.svg diff --git a/content/chapters/compute/lecture/media/resource-management.svg b/chapters/compute/processes/media/resource-management.svg similarity index 100% rename from content/chapters/compute/lecture/media/resource-management.svg rename to chapters/compute/processes/media/resource-management.svg diff --git a/content/chapters/compute/lecture/media/vas.svg b/chapters/compute/processes/media/vas.svg similarity index 100% rename from content/chapters/compute/lecture/media/vas.svg rename to chapters/compute/processes/media/vas.svg diff --git a/chapters/compute/processes/reading/processes.md b/chapters/compute/processes/reading/processes.md new file mode 100644 index 0000000000..4a42cbc4c4 --- /dev/null +++ b/chapters/compute/processes/reading/processes.md @@ -0,0 +1,84 @@ +# Processes + +A process is simply a running program. +Let's take the `ls` command as a trivial example. +`ls` is a **program** on your system. +It has a binary file which you can find and inspect with the help of the `which` command: + +```console +student@os:~$ which ls +/usr/bin/ls + +student@os:~$ file /usr/bin/ls +/usr/bin/ls: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=6e3da6f0bc36b6398b8651bbc2e08831a21a90da, for GNU/Linux 3.2.0, stripped +``` + +When you run it, the `ls` binary stored **on the disk** at `/usr/bin/ls` is read by another application called the **loader**. +The loader spawns a **process** by copying some of the contents `/usr/bin/ls` in memory (such as the `.text`, `.rodata` and `.data` sections). +Using `strace`, we can see the [`execve`](https://man7.org/linux/man-pages/man2/execve.2.html) system call: + +```console +student@os:~$ strace -s 100 ls -a # -s 100 limits strings to 100 bytes instead of the default 32 +execve("/usr/bin/ls", ["ls", "-a"], 0x7fffa7e0d008 /* 61 vars */) = 0 +[...] +write(1, ". .. content\tCONTRIBUTING.md COPYING.md .git .gitignore README.md REVIEWING.md\n", 86. .. content CONTRIBUTING.md COPYING.md .git .gitignore README.md REVIEWING.md +) = 86 +close(1) = 0 +close(2) = 0 +exit_group(0) = ? ++++ exited with 0 +++ +``` + +Look at its parameters: + +- the path to the **program**: `/usr/bin/ls` +- the list of arguments: `"ls", "-a"` +- the environment variables: the rest of the syscall's arguments + +`execve` invokes the loader to load the VAS of the `ls` process **by replacing that of the existing process**. +All subsequent syscalls are performed by the newly spawned `ls` process. +We will get into more details regarding `execve` [towards the end of this lab](./guides/system-dissected.md). + +![Loading of `ls` Process](../media/loading-of-ls-process.svg) + +## Fork + +Up to now we've been creating processes using various high-level APIs, such as `Popen()`, `Process()` and `system()`. +Yes, despite being a C function, as you've seen from its man page, `system()` itself calls 2 other functions: `fork()` to create a process and `execve()` to execute the given command. +As you already know from the [Software Stack](reading/libc.md) chapter, library functions may call one or more underlying system calls or other functions. +Now we will move one step lower on the call stack and call `fork()` ourselves. + +`fork()` creates one child process that is _almost_ identical to its parent. +We say that `fork()` returns **twice**: once in the parent process and once more in the child process. +This means that after `fork()` returns, assuming no error has occurred, both the child and the parent resume execution from the same place: the instruction following the call to `fork()`. +What's different between the two processes is the value returned by `fork()`: + +- **child process**: `fork()` returns 0 +- **parent process**: `fork()` returns the PID of the child process (> 0) +- **on error**: `fork()` returns -1, only once, in the initial process + +Therefore, the typical code for handling a `fork()` is available in `create-process/support/fork.c`. +Take a look at it and then run it. +Notice what each of the two processes prints: + +- the PID of the child is also known by the parent +- the PPID of the child is the PID of the parent + +Unlike `system()`, who also waits for its child, when using `fork()` we must do the waiting ourselves. +In order to wait for a process to end, we use the [`waitpid()`](https://linux.die.net/man/2/waitpid) syscall. +It places the exit code of the child process in the `status` parameter. +This argument is actually a bit-field containing more information than merely the exit code. +To retrieve the exit code, we use the `WEXITSTATUS` macro. +Keep in mind that `WEXITSTATUS` only makes sense if `WIFEXITED` is true, i.e. if the child process finished on its own and wasn't killed by another one or by an illegal action (such as a segfault or illegal instruction) for example. +Otherwise, `WEXITSTATUS` will return something meaningless. +You can view the rest of the information stored in the `status` bit-field [in the man page](https://linux.die.net/man/2/waitpid). + +**Moral of the story**: Usually the execution flow is: + +1. `fork()`, followed by + +1. `wait()` (called by the parent) + +1. `exit()`, called by the child. + +The order of last 2 steps may be swapped. diff --git a/content/chapters/compute/lecture/slides/fork.md b/chapters/compute/processes/slides/fork.md similarity index 81% rename from content/chapters/compute/lecture/slides/fork.md rename to chapters/compute/processes/slides/fork.md index cd39bb3ac4..be7bf8b377 100644 --- a/content/chapters/compute/lecture/slides/fork.md +++ b/chapters/compute/processes/slides/fork.md @@ -11,7 +11,7 @@ * Returns twice * `0` in child process * Child's PID in parent process -* [Quiz](../quiz/process-creation.md) +* [Quiz](../drills/questions/process-creation.md) * Initially child and parent share the same PAS (Physical Address Space) * Child's writable pages are marked **copy-on-write** @@ -22,4 +22,4 @@ * `fork` also creates a new thread... * ... but part of a different process -![fork - new Thread](./media/fork-thread.svg) +![fork - new Thread](../media/fork-thread.svg) diff --git a/content/chapters/compute/lecture/slides/process-attributes.md b/chapters/compute/processes/slides/process-attributes.md similarity index 100% rename from content/chapters/compute/lecture/slides/process-attributes.md rename to chapters/compute/processes/slides/process-attributes.md diff --git a/content/chapters/compute/lecture/slides/processes.md b/chapters/compute/processes/slides/processes.md similarity index 85% rename from content/chapters/compute/lecture/slides/processes.md rename to chapters/compute/processes/slides/processes.md index 4bac3360fa..56a7f1f18b 100644 --- a/content/chapters/compute/lecture/slides/processes.md +++ b/chapters/compute/processes/slides/processes.md @@ -9,7 +9,7 @@ * A process is a **running program** * It is a collection of resources (Memory, CPU, I/O) and abstractions over them (VAS, threads, file descriptors) * An application can spawn multiple processes -* The OS allows each [thread](./threads.md) to run on a **core** for a few milliseconds (time slice) +* The OS allows each [thread](../../threads/reading/threads.md) to run on a **core** for a few milliseconds (time slice) * Then the OS pauses the thread and replaces it with another one (context switch) ---- @@ -26,7 +26,7 @@ #### Resource Management - Overview -![Resource Management](media/resource-management.svg) +![Resource Management](../media/resource-management.svg) ---- @@ -48,7 +48,7 @@ #### Process Virtual Address Space -![Process VAS](media/vas.svg) +![Process VAS](../media/vas.svg) --- diff --git a/content/chapters/compute/lecture/quiz/number-of-running-threads.md b/chapters/compute/scheduling/drills/questions/number-of-running-threads.md similarity index 100% rename from content/chapters/compute/lecture/quiz/number-of-running-threads.md rename to chapters/compute/scheduling/drills/questions/number-of-running-threads.md diff --git a/content/chapters/compute/lab/quiz/number-of-running-ults.md b/chapters/compute/scheduling/drills/questions/number-of-running-ults.md similarity index 100% rename from content/chapters/compute/lab/quiz/number-of-running-ults.md rename to chapters/compute/scheduling/drills/questions/number-of-running-ults.md diff --git a/content/chapters/compute/lab/quiz/tcb-libult-unikraft.md b/chapters/compute/scheduling/drills/questions/tcb-libult-unikraft.md similarity index 100% rename from content/chapters/compute/lab/quiz/tcb-libult-unikraft.md rename to chapters/compute/scheduling/drills/questions/tcb-libult-unikraft.md diff --git a/content/chapters/compute/lab/quiz/time-slice-value.md b/chapters/compute/scheduling/drills/questions/time-slice-value.md similarity index 100% rename from content/chapters/compute/lab/quiz/time-slice-value.md rename to chapters/compute/scheduling/drills/questions/time-slice-value.md diff --git a/content/chapters/compute/lab/quiz/type-of-scheduler-in-libult.md b/chapters/compute/scheduling/drills/questions/type-of-scheduler-in-libult.md similarity index 100% rename from content/chapters/compute/lab/quiz/type-of-scheduler-in-libult.md rename to chapters/compute/scheduling/drills/questions/type-of-scheduler-in-libult.md diff --git a/content/chapters/compute/lab/quiz/ult-thread-ids.md b/chapters/compute/scheduling/drills/questions/ult-thread-ids.md similarity index 100% rename from content/chapters/compute/lab/quiz/ult-thread-ids.md rename to chapters/compute/scheduling/drills/questions/ult-thread-ids.md diff --git a/content/chapters/compute/lab/quiz/why-use-completed-queue.md b/chapters/compute/scheduling/drills/questions/why-use-completed-queue.md similarity index 100% rename from content/chapters/compute/lab/quiz/why-use-completed-queue.md rename to chapters/compute/scheduling/drills/questions/why-use-completed-queue.md diff --git a/chapters/compute/scheduling/drills/tasks/libult/README.md b/chapters/compute/scheduling/drills/tasks/libult/README.md new file mode 100644 index 0000000000..b3572a165f --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/README.md @@ -0,0 +1,11 @@ +# Another Time Slice + +Enter the `libult/support/` folder and go through the practice items below. + +1. Modify the time slice set to the timer to 2 seconds. +Re-run the code in `libult/support/test_ult.c`. +Notice that now no context switch happens between the 2 created threads because they end before the timer can fire. + +1. Now change the `printer_thread()` function in `test_ult.c` to make it run for more than 2 seconds. +See that now the prints from the two threads appear intermingled. +Add prints to the `handle_sigprof()` function in `libult/support/threads.c` to see the context switch happen. diff --git a/content/chapters/compute/lab/solution/libult/.gitignore b/chapters/compute/scheduling/drills/tasks/libult/solution/.gitignore similarity index 100% rename from content/chapters/compute/lab/solution/libult/.gitignore rename to chapters/compute/scheduling/drills/tasks/libult/solution/.gitignore diff --git a/content/chapters/compute/lab/support/libult/CPPLINT.cfg b/chapters/compute/scheduling/drills/tasks/libult/solution/CPPLINT.cfg similarity index 100% rename from content/chapters/compute/lab/support/libult/CPPLINT.cfg rename to chapters/compute/scheduling/drills/tasks/libult/solution/CPPLINT.cfg diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/Makefile b/chapters/compute/scheduling/drills/tasks/libult/solution/Makefile new file mode 100644 index 0000000000..8b1fb2a9e3 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/Makefile @@ -0,0 +1,47 @@ +TEST = test_ult +LIBULT = libult.so + +all: $(TEST) $(LIBULT) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +LIB_OBJECTS = queue.o tcb.o threads.o +TEST_OBJECTS = test_ult.o $(LOGGER) +LDFLAGS = -L. +LDLIBS = -lult + +$(TEST): $(TEST_OBJECTS) $(LIBULT) + $(CC) -o $@ $(TEST_OBJECTS) $(LDFLAGS) $(LDLIBS) + +$(LIBULT): $(LIB_OBJECTS) + $(CC) -shared -o $@ $^ + +clean:: + rm -f $(TEST) $(LIBULT) + +.PHONY: all clean diff --git a/content/chapters/compute/lab/support/libult/queue.c b/chapters/compute/scheduling/drills/tasks/libult/solution/queue.c similarity index 50% rename from content/chapters/compute/lab/support/libult/queue.c rename to chapters/compute/scheduling/drills/tasks/libult/solution/queue.c index 355c03b7aa..c887b4af72 100644 --- a/content/chapters/compute/lab/support/libult/queue.c +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/queue.c @@ -1,26 +1,28 @@ -/** -* MIT License -* -* Copyright (c) 2020 Andreas Schärtl and Contributors -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ /* Github link: https://github.com/kissen/threads */ @@ -42,16 +44,12 @@ struct queue { size_t size; }; - - - QUEUE *queue_new(void) { - QUEUE *new; + QUEUE *new = calloc(1, sizeof(QUEUE)); - if ((new = calloc(1, sizeof(QUEUE))) == NULL) { + if (new == NULL) return NULL; - } return new; } @@ -84,11 +82,10 @@ int queue_enqueue(QUEUE *queue, TCB *elem) { // Create the new node - struct node *new; + struct node *new = malloc(sizeof(struct node)); - if ((new = malloc(sizeof(struct node))) == NULL) { - return errno; - } + if (new == NULL) + return errno; new->thread = elem; new->next = NULL; @@ -99,9 +96,9 @@ int queue_enqueue(QUEUE *queue, TCB *elem) queue->head = new; } else { struct node *parent = queue->head; - while (parent->next != NULL) { + + while (parent->next != NULL) parent = parent->next; - } parent->next = new; } @@ -113,13 +110,14 @@ int queue_enqueue(QUEUE *queue, TCB *elem) TCB *queue_dequeue(QUEUE *queue) { struct node *old_head = queue->head; - if (old_head == NULL || queue->size == 0) { + + if (old_head == NULL || queue->size == 0) return NULL; - } queue->head = queue->head->next; queue->size -= 1; TCB *retval = old_head->thread; + free(old_head); return retval; @@ -128,22 +126,21 @@ TCB *queue_dequeue(QUEUE *queue) TCB *queue_remove_id(QUEUE *queue, int id) { - if (queue->head == NULL) { + if (queue->head == NULL) return NULL; - } struct node *prev = NULL; struct node *cur = queue->head; while (cur != NULL) { if (cur->thread->id == id) { - if (prev == NULL) { + if (prev == NULL) queue->head = cur->next; - } else { + else prev->next = cur->next; - } TCB *retval = cur->thread; + free(cur); return retval; } diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/queue.h b/chapters/compute/scheduling/drills/tasks/libult/solution/queue.h new file mode 100644 index 0000000000..022d990829 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/queue.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: MIT */ + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Defines a queue to mange TCB elements. + */ + +#ifndef QUEUE_H +#define QUEUE_H + + +#include "tcb.h" + +#include + + +typedef struct queue QUEUE; + + +/* Create a new initialized QUEUE on the heap. Returns a pointer to + * the new block or NULL on error. + */ +QUEUE *queue_new(void); + + +/* Destroy queue, freeing all associated memory with it. It also frees + * all memory of the elements inside the queue. + */ +void queue_destroy(QUEUE *queue); + + +/* Return the number of items in queue. */ +size_t queue_size(const QUEUE *queue); + + +/* Add elem to the end of queue. Returns 0 on succes and non-zero on + * failure. + */ +int queue_enqueue(QUEUE *queue, TCB *elem); + + +/* Remove the first item from the queue and return it. The caller will + * have to free the reuturned element. Returns NULL if the queue is + * empty. + */ +TCB *queue_dequeue(QUEUE *queue); + + +/* Remove element with matching id from the queue. Returns the removed + * element or NULL if no such element exists. + */ +TCB *queue_remove_id(QUEUE *queue, int id); + + +#endif diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/tcb.c b/chapters/compute/scheduling/drills/tasks/libult/solution/tcb.c new file mode 100644 index 0000000000..00d8b26cc7 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/tcb.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +#include "tcb.h" + +#include + + +TCB *tcb_new(void) +{ + static int next_id = 1; + + TCB *new = calloc(1, sizeof(TCB)); + + if (new == NULL) + return NULL; + + new->id = next_id++; + return new; +} + + +void tcb_destroy(TCB *block) +{ + if (block->has_dynamic_stack) + free(block->context.uc_stack.ss_sp); + + free(block); +} diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/tcb.h b/chapters/compute/scheduling/drills/tasks/libult/solution/tcb.h new file mode 100644 index 0000000000..ac2e19f00f --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/tcb.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ + +/** + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Defines the TCB (thread control block) data structure and functions + * to work with it. + */ + +#ifndef TCB_H +#define TCB_H + + +#include +#include +#include + + +typedef struct { + int id; + ucontext_t context; + bool has_dynamic_stack; + void *(*start_routine)(void *arg); + void *argument; + void *return_value; +} TCB; + + +/* Create a new zeroed TCB on the heap. Returns a pointer to the new + * block or NULL on error. + */ +TCB *tcb_new(void); + + +/* Destroy block, freeing all associated memory with it */ +void tcb_destroy(TCB *block); + + +#endif diff --git a/content/chapters/compute/lab/support/libult/test_ult.c b/chapters/compute/scheduling/drills/tasks/libult/solution/test_ult.c similarity index 83% rename from content/chapters/compute/lab/support/libult/test_ult.c rename to chapters/compute/scheduling/drills/tasks/libult/solution/test_ult.c index c57fbe65e8..13366e762b 100644 --- a/content/chapters/compute/lab/support/libult/test_ult.c +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/test_ult.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: BSD-3-Clause + #include #include #include @@ -27,10 +29,10 @@ int main(void) void *res; int rc; - puts("Hello, this is main()."); + puts("Hello, this is the _main() func."); for (i = 0; i < NUM_THREADS; ++i) { - threads[i] = threads_create(printer_thread, (void*)i); + threads[i] = threads_create(printer_thread, (void *)i); DIE(threads[i] < 0, "threads_create"); } diff --git a/content/chapters/compute/lab/support/libult/threads.c b/chapters/compute/scheduling/drills/tasks/libult/solution/threads.c similarity index 65% rename from content/chapters/compute/lab/support/libult/threads.c rename to chapters/compute/scheduling/drills/tasks/libult/solution/threads.c index 83efead8f0..a4f77cba48 100644 --- a/content/chapters/compute/lab/support/libult/threads.c +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/threads.c @@ -1,26 +1,28 @@ -/** -* MIT License -* -* Copyright (c) 2020 Andreas Schärtl and Contributors -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ /* Github link: https://github.com/kissen/threads */ @@ -39,7 +41,7 @@ #include -static QUEUE *ready, *completed; +static struct queue *ready, *completed; static TCB *running; @@ -48,7 +50,7 @@ static bool init_first_context(void); static bool init_profiling_timer(void); static void handle_sigprof(int, siginfo_t *, void *); -static void handle_thread_start(); +static void handle_thread_start(void); static bool malloc_stack(TCB *); @@ -64,36 +66,32 @@ int threads_create(void *(*start_routine) (void *), void *arg) static bool initialized; - if (! initialized) { - if (! init_queues()) { + if (!initialized) { + if (!init_queues()) abort(); - } - if (! init_first_context()) { + if (!init_first_context()) abort(); - } - if (! init_profiling_timer()) { + if (!init_profiling_timer()) abort(); - } - + initialized = true; } // Create a thread control block for the newly created thread. - TCB *new; + TCB *new = tcb_new(); - if ((new = tcb_new()) == NULL) { + if (new == NULL) return -1; - } if (getcontext(&new->context) == -1) { tcb_destroy(new); return -1; } - if (! malloc_stack(new)) { + if (!malloc_stack(new)) { tcb_destroy(new); return -1; } @@ -117,20 +115,18 @@ int threads_create(void *(*start_routine) (void *), void *arg) void threads_exit(void *result) { - if (running == NULL) { + if (running == NULL) exit(EXIT_SUCCESS); - } block_sigprof(); running->return_value = result; - if (queue_enqueue(completed, running) != 0) { + if (queue_enqueue(completed, running) != 0) abort(); - } - if ((running = queue_dequeue(ready)) == NULL) { + running = queue_dequeue(ready); + if (running == NULL) exit(EXIT_SUCCESS); - } setcontext(&running->context); // also unblocks SIGPROF } @@ -145,25 +141,26 @@ int threads_join(int id, void **result) block_sigprof(); TCB *block = queue_remove_id(completed, id); + unblock_sigprof(); - if (block == NULL) { + if (block == NULL) return 0; - } else { - *result = block->return_value; - tcb_destroy(block); - return id; - } + + *result = block->return_value; + tcb_destroy(block); + return id; } static bool init_queues(void) { - if ((ready = queue_new()) == NULL) { + ready = queue_new(); + if (ready == NULL) return false; - } - if ((completed = queue_new()) == NULL) { + completed = queue_new(); + if (completed == NULL) { queue_destroy(ready); return false; } @@ -173,11 +170,10 @@ static bool init_queues(void) static bool init_first_context(void) { - TCB *block; + TCB *block = tcb_new(); - if ((block = tcb_new()) == NULL) { + if (block == NULL) return false; - } if (getcontext(&block->context) == -1) { tcb_destroy(block); @@ -194,6 +190,7 @@ static bool init_profiling_timer(void) // Install signal handler sigset_t all; + sigfillset(&all); const struct sigaction alarm = { @@ -216,7 +213,7 @@ static bool init_profiling_timer(void) // Enable timer - if (setitimer(ITIMER_PROF, &timer, NULL) == - 1) { + if (setitimer(ITIMER_PROF, &timer, NULL) == -1) { if (sigaction(SIGPROF, &old, NULL) == -1) { perror("sigaction"); abort(); @@ -236,12 +233,11 @@ static void handle_sigprof(int signum, siginfo_t *nfo, void *context) (void)signum; (void)nfo; - if (running == NULL && queue_size(ready) == 0) { + if (running == NULL && queue_size(ready) == 0) _exit(EXIT_SUCCESS); - } // Backup the current context - + ucontext_t *stored = &running->context; ucontext_t *updated = (ucontext_t *) context; @@ -252,20 +248,18 @@ static void handle_sigprof(int signum, siginfo_t *nfo, void *context) // Round robin - if (queue_enqueue(ready, running) != 0) { + if (queue_enqueue(ready, running) != 0) abort(); - } - if ((running = queue_dequeue(ready)) == NULL) { + running = queue_dequeue(ready); + if (running == NULL) abort(); - } // Manually leave the signal handler errno = old_errno; - if (setcontext(&running->context) == -1) { + if (setcontext(&running->context) == -1) abort(); - } } @@ -273,30 +267,30 @@ static void handle_thread_start(void) { block_sigprof(); TCB *this = running; + unblock_sigprof(); void *result = this->start_routine(this->argument); + threads_exit(result); } - + static bool malloc_stack(TCB *thread) { // Get the stack size struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == -1) { + if (getrlimit(RLIMIT_STACK, &limit) == -1) return false; - } // Allocate memory - void *stack; + void *stack = malloc(limit.rlim_cur); - if ((stack = malloc(limit.rlim_cur)) == NULL) { + if (stack == NULL) return false; - } // Update the thread control bock @@ -312,6 +306,7 @@ static bool malloc_stack(TCB *thread) static void block_sigprof(void) { sigset_t sigprof; + sigemptyset(&sigprof); sigaddset(&sigprof, SIGPROF); @@ -325,6 +320,7 @@ static void block_sigprof(void) static void unblock_sigprof(void) { sigset_t sigprof; + sigemptyset(&sigprof); sigaddset(&sigprof, SIGPROF); diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/threads.h b/chapters/compute/scheduling/drills/tasks/libult/solution/threads.h new file mode 100644 index 0000000000..255a802e62 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/threads.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Interface to a barebones user level thread library. + */ + +#ifndef THREADS_H +#define THREADS_H + + +/* Create a new thread. func is the function that will be run once the + * thread starts execution and arg is the argument for that + * function. On success, returns an id equal or greater to 0. On + * failure, errno is set and -1 is returned. + */ +int threads_create(void *(*start_routine) (void *), void *arg); + + +/* Stop execution of the thread calling this function. */ +void threads_exit(void *result); + + +/* Wait for the thread with matching id to finish execution, that is, + * for it to call threads_exit. On success, the threads result is + * written into result and id is returned. If no completed thread with + * matching id exists, 0 is returned. On error, -1 is returned and + * errno is set. + */ +int threads_join(int id, void **result); + + +#endif diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/utils/log/CPPLINT.cfg b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/utils/log/log.c b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/utils/log/log.h b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.c b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.h b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/utils/utils.h b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/support/libult/.gitignore b/chapters/compute/scheduling/drills/tasks/libult/support/.gitignore similarity index 100% rename from content/chapters/compute/lab/support/libult/.gitignore rename to chapters/compute/scheduling/drills/tasks/libult/support/.gitignore diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/CPPLINT.cfg b/chapters/compute/scheduling/drills/tasks/libult/support/CPPLINT.cfg new file mode 100644 index 0000000000..ab07850428 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=(queue|tcb|threads)\.(c|h) diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/Makefile b/chapters/compute/scheduling/drills/tasks/libult/support/Makefile new file mode 100644 index 0000000000..8b1fb2a9e3 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/Makefile @@ -0,0 +1,47 @@ +TEST = test_ult +LIBULT = libult.so + +all: $(TEST) $(LIBULT) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +LIB_OBJECTS = queue.o tcb.o threads.o +TEST_OBJECTS = test_ult.o $(LOGGER) +LDFLAGS = -L. +LDLIBS = -lult + +$(TEST): $(TEST_OBJECTS) $(LIBULT) + $(CC) -o $@ $(TEST_OBJECTS) $(LDFLAGS) $(LDLIBS) + +$(LIBULT): $(LIB_OBJECTS) + $(CC) -shared -o $@ $^ + +clean:: + rm -f $(TEST) $(LIBULT) + +.PHONY: all clean diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/queue.c b/chapters/compute/scheduling/drills/tasks/libult/support/queue.c new file mode 100644 index 0000000000..c887b4af72 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/queue.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +#include "queue.h" + + +#include +#include + + +struct node { + TCB *thread; + struct node *next; +}; + + +struct queue { + struct node *head; + size_t size; +}; + +QUEUE *queue_new(void) +{ + QUEUE *new = calloc(1, sizeof(QUEUE)); + + if (new == NULL) + return NULL; + + return new; +} + + +void queue_destroy(QUEUE *queue) +{ + struct node *prev = NULL; + struct node *cursor = queue->head; + + while (cursor != NULL) { + prev = cursor; + cursor = cursor->next; + + tcb_destroy(prev->thread); + free(prev); + } + + free(queue); +} + + +size_t queue_size(const QUEUE *queue) +{ + return queue->size; +} + + +int queue_enqueue(QUEUE *queue, TCB *elem) +{ + // Create the new node + + struct node *new = malloc(sizeof(struct node)); + + if (new == NULL) + return errno; + + new->thread = elem; + new->next = NULL; + + // Enqueue the new node + + if (queue->head == NULL) { + queue->head = new; + } else { + struct node *parent = queue->head; + + while (parent->next != NULL) + parent = parent->next; + parent->next = new; + } + + queue->size += 1; + return 0; +} + + +TCB *queue_dequeue(QUEUE *queue) +{ + struct node *old_head = queue->head; + + if (old_head == NULL || queue->size == 0) + return NULL; + queue->head = queue->head->next; + queue->size -= 1; + + TCB *retval = old_head->thread; + + free(old_head); + + return retval; +} + + +TCB *queue_remove_id(QUEUE *queue, int id) +{ + if (queue->head == NULL) + return NULL; + + struct node *prev = NULL; + struct node *cur = queue->head; + + while (cur != NULL) { + if (cur->thread->id == id) { + if (prev == NULL) + queue->head = cur->next; + else + prev->next = cur->next; + + TCB *retval = cur->thread; + + free(cur); + return retval; + } + + prev = cur; + cur = cur->next; + } + + return NULL; +} diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/queue.h b/chapters/compute/scheduling/drills/tasks/libult/support/queue.h new file mode 100644 index 0000000000..022d990829 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/queue.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: MIT */ + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Defines a queue to mange TCB elements. + */ + +#ifndef QUEUE_H +#define QUEUE_H + + +#include "tcb.h" + +#include + + +typedef struct queue QUEUE; + + +/* Create a new initialized QUEUE on the heap. Returns a pointer to + * the new block or NULL on error. + */ +QUEUE *queue_new(void); + + +/* Destroy queue, freeing all associated memory with it. It also frees + * all memory of the elements inside the queue. + */ +void queue_destroy(QUEUE *queue); + + +/* Return the number of items in queue. */ +size_t queue_size(const QUEUE *queue); + + +/* Add elem to the end of queue. Returns 0 on succes and non-zero on + * failure. + */ +int queue_enqueue(QUEUE *queue, TCB *elem); + + +/* Remove the first item from the queue and return it. The caller will + * have to free the reuturned element. Returns NULL if the queue is + * empty. + */ +TCB *queue_dequeue(QUEUE *queue); + + +/* Remove element with matching id from the queue. Returns the removed + * element or NULL if no such element exists. + */ +TCB *queue_remove_id(QUEUE *queue, int id); + + +#endif diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/tcb.c b/chapters/compute/scheduling/drills/tasks/libult/support/tcb.c new file mode 100644 index 0000000000..00d8b26cc7 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/tcb.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +#include "tcb.h" + +#include + + +TCB *tcb_new(void) +{ + static int next_id = 1; + + TCB *new = calloc(1, sizeof(TCB)); + + if (new == NULL) + return NULL; + + new->id = next_id++; + return new; +} + + +void tcb_destroy(TCB *block) +{ + if (block->has_dynamic_stack) + free(block->context.uc_stack.ss_sp); + + free(block); +} diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/tcb.h b/chapters/compute/scheduling/drills/tasks/libult/support/tcb.h new file mode 100644 index 0000000000..ac2e19f00f --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/tcb.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ + +/** + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Defines the TCB (thread control block) data structure and functions + * to work with it. + */ + +#ifndef TCB_H +#define TCB_H + + +#include +#include +#include + + +typedef struct { + int id; + ucontext_t context; + bool has_dynamic_stack; + void *(*start_routine)(void *arg); + void *argument; + void *return_value; +} TCB; + + +/* Create a new zeroed TCB on the heap. Returns a pointer to the new + * block or NULL on error. + */ +TCB *tcb_new(void); + + +/* Destroy block, freeing all associated memory with it */ +void tcb_destroy(TCB *block); + + +#endif diff --git a/content/chapters/compute/lab/solution/libult/test_ult.c b/chapters/compute/scheduling/drills/tasks/libult/support/test_ult.c similarity index 72% rename from content/chapters/compute/lab/solution/libult/test_ult.c rename to chapters/compute/scheduling/drills/tasks/libult/support/test_ult.c index 71387a8ca6..13366e762b 100644 --- a/content/chapters/compute/lab/solution/libult/test_ult.c +++ b/chapters/compute/scheduling/drills/tasks/libult/support/test_ult.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: BSD-3-Clause + #include #include #include @@ -5,17 +7,17 @@ #include "./threads.h" #include "utils/utils.h" -#define NUM_ITER 10000000000 +#define NUM_ITER 10000000 #define NUM_THREADS 2 static void *printer_thread(void *arg) { size_t id = (size_t)arg; - size_t i; + int i; for (i = 0; i < NUM_ITER; ++i) - if (!(i % 1000000000)) - printf("Thread %zu: i = %zu\n", id, i); + if (!(i % 1000000)) + printf("Thread %zu: i = %d\n", id, i); return NULL; } @@ -27,10 +29,10 @@ int main(void) void *res; int rc; - puts("Hello, this is main()."); + puts("Hello, this is the _main() func."); for (i = 0; i < NUM_THREADS; ++i) { - threads[i] = threads_create(printer_thread, (void*)i); + threads[i] = threads_create(printer_thread, (void *)i); DIE(threads[i] < 0, "threads_create"); } diff --git a/content/chapters/compute/lab/solution/libult/threads.c b/chapters/compute/scheduling/drills/tasks/libult/support/threads.c similarity index 63% rename from content/chapters/compute/lab/solution/libult/threads.c rename to chapters/compute/scheduling/drills/tasks/libult/support/threads.c index bccf493cad..a4f77cba48 100644 --- a/content/chapters/compute/lab/solution/libult/threads.c +++ b/chapters/compute/scheduling/drills/tasks/libult/support/threads.c @@ -1,26 +1,28 @@ -/** -* MIT License -* -* Copyright (c) 2020 Andreas Schärtl and Contributors -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ /* Github link: https://github.com/kissen/threads */ @@ -39,7 +41,7 @@ #include -static QUEUE *ready, *completed; +static struct queue *ready, *completed; static TCB *running; @@ -48,7 +50,7 @@ static bool init_first_context(void); static bool init_profiling_timer(void); static void handle_sigprof(int, siginfo_t *, void *); -static void handle_thread_start(); +static void handle_thread_start(void); static bool malloc_stack(TCB *); @@ -64,36 +66,32 @@ int threads_create(void *(*start_routine) (void *), void *arg) static bool initialized; - if (! initialized) { - if (! init_queues()) { + if (!initialized) { + if (!init_queues()) abort(); - } - if (! init_first_context()) { + if (!init_first_context()) abort(); - } - if (! init_profiling_timer()) { + if (!init_profiling_timer()) abort(); - } - + initialized = true; } // Create a thread control block for the newly created thread. - TCB *new; + TCB *new = tcb_new(); - if ((new = tcb_new()) == NULL) { + if (new == NULL) return -1; - } if (getcontext(&new->context) == -1) { tcb_destroy(new); return -1; } - if (! malloc_stack(new)) { + if (!malloc_stack(new)) { tcb_destroy(new); return -1; } @@ -117,20 +115,18 @@ int threads_create(void *(*start_routine) (void *), void *arg) void threads_exit(void *result) { - if (running == NULL) { + if (running == NULL) exit(EXIT_SUCCESS); - } block_sigprof(); running->return_value = result; - if (queue_enqueue(completed, running) != 0) { + if (queue_enqueue(completed, running) != 0) abort(); - } - if ((running = queue_dequeue(ready)) == NULL) { + running = queue_dequeue(ready); + if (running == NULL) exit(EXIT_SUCCESS); - } setcontext(&running->context); // also unblocks SIGPROF } @@ -145,25 +141,26 @@ int threads_join(int id, void **result) block_sigprof(); TCB *block = queue_remove_id(completed, id); + unblock_sigprof(); - if (block == NULL) { + if (block == NULL) return 0; - } else { - *result = block->return_value; - tcb_destroy(block); - return id; - } + + *result = block->return_value; + tcb_destroy(block); + return id; } static bool init_queues(void) { - if ((ready = queue_new()) == NULL) { + ready = queue_new(); + if (ready == NULL) return false; - } - if ((completed = queue_new()) == NULL) { + completed = queue_new(); + if (completed == NULL) { queue_destroy(ready); return false; } @@ -173,11 +170,10 @@ static bool init_queues(void) static bool init_first_context(void) { - TCB *block; + TCB *block = tcb_new(); - if ((block = tcb_new()) == NULL) { + if (block == NULL) return false; - } if (getcontext(&block->context) == -1) { tcb_destroy(block); @@ -194,6 +190,7 @@ static bool init_profiling_timer(void) // Install signal handler sigset_t all; + sigfillset(&all); const struct sigaction alarm = { @@ -210,13 +207,13 @@ static bool init_profiling_timer(void) } const struct itimerval timer = { - { 2, 0 }, + { 0, 10000 }, { 0, 1 } // arms the timer as soon as possible }; // Enable timer - if (setitimer(ITIMER_PROF, &timer, NULL) == - 1) { + if (setitimer(ITIMER_PROF, &timer, NULL) == -1) { if (sigaction(SIGPROF, &old, NULL) == -1) { perror("sigaction"); abort(); @@ -236,12 +233,11 @@ static void handle_sigprof(int signum, siginfo_t *nfo, void *context) (void)signum; (void)nfo; - if (running == NULL && queue_size(ready) == 0) { + if (running == NULL && queue_size(ready) == 0) _exit(EXIT_SUCCESS); - } // Backup the current context - + ucontext_t *stored = &running->context; ucontext_t *updated = (ucontext_t *) context; @@ -252,24 +248,18 @@ static void handle_sigprof(int signum, siginfo_t *nfo, void *context) // Round robin - printf("[scheduler] Preempting running thread with id %d\n", running->id); - - if (queue_enqueue(ready, running) != 0) { + if (queue_enqueue(ready, running) != 0) abort(); - } - if ((running = queue_dequeue(ready)) == NULL) { + running = queue_dequeue(ready); + if (running == NULL) abort(); - } - - printf("[scheduler] Running new thread with id %d\n", running->id); // Manually leave the signal handler errno = old_errno; - if (setcontext(&running->context) == -1) { + if (setcontext(&running->context) == -1) abort(); - } } @@ -277,30 +267,30 @@ static void handle_thread_start(void) { block_sigprof(); TCB *this = running; + unblock_sigprof(); void *result = this->start_routine(this->argument); + threads_exit(result); } - + static bool malloc_stack(TCB *thread) { // Get the stack size struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == -1) { + if (getrlimit(RLIMIT_STACK, &limit) == -1) return false; - } // Allocate memory - void *stack; + void *stack = malloc(limit.rlim_cur); - if ((stack = malloc(limit.rlim_cur)) == NULL) { + if (stack == NULL) return false; - } // Update the thread control bock @@ -316,6 +306,7 @@ static bool malloc_stack(TCB *thread) static void block_sigprof(void) { sigset_t sigprof; + sigemptyset(&sigprof); sigaddset(&sigprof, SIGPROF); @@ -329,6 +320,7 @@ static void block_sigprof(void) static void unblock_sigprof(void) { sigset_t sigprof; + sigemptyset(&sigprof); sigaddset(&sigprof, SIGPROF); diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/threads.h b/chapters/compute/scheduling/drills/tasks/libult/support/threads.h new file mode 100644 index 0000000000..255a802e62 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/threads.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Interface to a barebones user level thread library. + */ + +#ifndef THREADS_H +#define THREADS_H + + +/* Create a new thread. func is the function that will be run once the + * thread starts execution and arg is the argument for that + * function. On success, returns an id equal or greater to 0. On + * failure, errno is set and -1 is returned. + */ +int threads_create(void *(*start_routine) (void *), void *arg); + + +/* Stop execution of the thread calling this function. */ +void threads_exit(void *result); + + +/* Wait for the thread with matching id to finish execution, that is, + * for it to call threads_exit. On success, the threads result is + * written into result and id is returned. If no completed thread with + * matching id exists, 0 is returned. On error, -1 is returned and + * errno is set. + */ +int threads_join(int id, void **result); + + +#endif diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/utils/log/CPPLINT.cfg b/chapters/compute/scheduling/drills/tasks/libult/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/utils/log/log.c b/chapters/compute/scheduling/drills/tasks/libult/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/utils/log/log.h b/chapters/compute/scheduling/drills/tasks/libult/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.c b/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.h b/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/utils/utils.h b/chapters/compute/scheduling/drills/tasks/libult/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/scheduling/drills/tasks/libult/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/scheduling/guides/libult/README.md b/chapters/compute/scheduling/guides/libult/README.md new file mode 100644 index 0000000000..36b4384757 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/README.md @@ -0,0 +1,26 @@ +# User-Level Threads Scheduler + +Go to `libult/support`. +It contains a minimalist **user-level threads** scheduler. +Compiling it produces a shared library called `libult.so`. +You can also consult its [documentation](https://www.schaertl.me/posts/a-bare-bones-user-level-thread-library/). +It explains the API as well as its implementation. +The API exposed by the scheduling library is very simple. +It is only made up of 3 functions: + +- `threads_create()` creates a new ULT +- `threads_exit()` moves the current ULT to the COMPLETED state +- `threads_join()` waits for a given thread to end and saves its return value in the `result` argument + +Look inside `libult/support/threads.c`. +Here you will find the 3 functions mentioned above. + +The scheduler only uses 3 states: RUNNING, READY, COMPLETED. + +[Quiz](../../drills/questions/number-of-running-ults.md) + +The threads in the READY and COMPLETED states are kept in 2 separate queues. +When the scheduler wants to run a new thread, it retrieves it from the READY queue. +When a thread ends its execution, it is added to the COMPLETED queue, together with its return value. + +[Quiz](../../drills/questions/why-use-completed-queue.md) diff --git a/chapters/compute/scheduling/guides/libult/solution/.gitignore b/chapters/compute/scheduling/guides/libult/solution/.gitignore new file mode 100644 index 0000000000..7cb003edb1 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/.gitignore @@ -0,0 +1 @@ +test_ult diff --git a/chapters/compute/scheduling/guides/libult/solution/CPPLINT.cfg b/chapters/compute/scheduling/guides/libult/solution/CPPLINT.cfg new file mode 100644 index 0000000000..ab07850428 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=(queue|tcb|threads)\.(c|h) diff --git a/chapters/compute/scheduling/guides/libult/solution/Makefile b/chapters/compute/scheduling/guides/libult/solution/Makefile new file mode 100644 index 0000000000..8b1fb2a9e3 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/Makefile @@ -0,0 +1,47 @@ +TEST = test_ult +LIBULT = libult.so + +all: $(TEST) $(LIBULT) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +LIB_OBJECTS = queue.o tcb.o threads.o +TEST_OBJECTS = test_ult.o $(LOGGER) +LDFLAGS = -L. +LDLIBS = -lult + +$(TEST): $(TEST_OBJECTS) $(LIBULT) + $(CC) -o $@ $(TEST_OBJECTS) $(LDFLAGS) $(LDLIBS) + +$(LIBULT): $(LIB_OBJECTS) + $(CC) -shared -o $@ $^ + +clean:: + rm -f $(TEST) $(LIBULT) + +.PHONY: all clean diff --git a/chapters/compute/scheduling/guides/libult/solution/queue.c b/chapters/compute/scheduling/guides/libult/solution/queue.c new file mode 100644 index 0000000000..c887b4af72 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/queue.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +#include "queue.h" + + +#include +#include + + +struct node { + TCB *thread; + struct node *next; +}; + + +struct queue { + struct node *head; + size_t size; +}; + +QUEUE *queue_new(void) +{ + QUEUE *new = calloc(1, sizeof(QUEUE)); + + if (new == NULL) + return NULL; + + return new; +} + + +void queue_destroy(QUEUE *queue) +{ + struct node *prev = NULL; + struct node *cursor = queue->head; + + while (cursor != NULL) { + prev = cursor; + cursor = cursor->next; + + tcb_destroy(prev->thread); + free(prev); + } + + free(queue); +} + + +size_t queue_size(const QUEUE *queue) +{ + return queue->size; +} + + +int queue_enqueue(QUEUE *queue, TCB *elem) +{ + // Create the new node + + struct node *new = malloc(sizeof(struct node)); + + if (new == NULL) + return errno; + + new->thread = elem; + new->next = NULL; + + // Enqueue the new node + + if (queue->head == NULL) { + queue->head = new; + } else { + struct node *parent = queue->head; + + while (parent->next != NULL) + parent = parent->next; + parent->next = new; + } + + queue->size += 1; + return 0; +} + + +TCB *queue_dequeue(QUEUE *queue) +{ + struct node *old_head = queue->head; + + if (old_head == NULL || queue->size == 0) + return NULL; + queue->head = queue->head->next; + queue->size -= 1; + + TCB *retval = old_head->thread; + + free(old_head); + + return retval; +} + + +TCB *queue_remove_id(QUEUE *queue, int id) +{ + if (queue->head == NULL) + return NULL; + + struct node *prev = NULL; + struct node *cur = queue->head; + + while (cur != NULL) { + if (cur->thread->id == id) { + if (prev == NULL) + queue->head = cur->next; + else + prev->next = cur->next; + + TCB *retval = cur->thread; + + free(cur); + return retval; + } + + prev = cur; + cur = cur->next; + } + + return NULL; +} diff --git a/chapters/compute/scheduling/guides/libult/solution/queue.h b/chapters/compute/scheduling/guides/libult/solution/queue.h new file mode 100644 index 0000000000..022d990829 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/queue.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: MIT */ + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Defines a queue to mange TCB elements. + */ + +#ifndef QUEUE_H +#define QUEUE_H + + +#include "tcb.h" + +#include + + +typedef struct queue QUEUE; + + +/* Create a new initialized QUEUE on the heap. Returns a pointer to + * the new block or NULL on error. + */ +QUEUE *queue_new(void); + + +/* Destroy queue, freeing all associated memory with it. It also frees + * all memory of the elements inside the queue. + */ +void queue_destroy(QUEUE *queue); + + +/* Return the number of items in queue. */ +size_t queue_size(const QUEUE *queue); + + +/* Add elem to the end of queue. Returns 0 on succes and non-zero on + * failure. + */ +int queue_enqueue(QUEUE *queue, TCB *elem); + + +/* Remove the first item from the queue and return it. The caller will + * have to free the reuturned element. Returns NULL if the queue is + * empty. + */ +TCB *queue_dequeue(QUEUE *queue); + + +/* Remove element with matching id from the queue. Returns the removed + * element or NULL if no such element exists. + */ +TCB *queue_remove_id(QUEUE *queue, int id); + + +#endif diff --git a/chapters/compute/scheduling/guides/libult/solution/tcb.c b/chapters/compute/scheduling/guides/libult/solution/tcb.c new file mode 100644 index 0000000000..00d8b26cc7 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/tcb.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +#include "tcb.h" + +#include + + +TCB *tcb_new(void) +{ + static int next_id = 1; + + TCB *new = calloc(1, sizeof(TCB)); + + if (new == NULL) + return NULL; + + new->id = next_id++; + return new; +} + + +void tcb_destroy(TCB *block) +{ + if (block->has_dynamic_stack) + free(block->context.uc_stack.ss_sp); + + free(block); +} diff --git a/chapters/compute/scheduling/guides/libult/solution/tcb.h b/chapters/compute/scheduling/guides/libult/solution/tcb.h new file mode 100644 index 0000000000..ac2e19f00f --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/tcb.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ + +/** + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Defines the TCB (thread control block) data structure and functions + * to work with it. + */ + +#ifndef TCB_H +#define TCB_H + + +#include +#include +#include + + +typedef struct { + int id; + ucontext_t context; + bool has_dynamic_stack; + void *(*start_routine)(void *arg); + void *argument; + void *return_value; +} TCB; + + +/* Create a new zeroed TCB on the heap. Returns a pointer to the new + * block or NULL on error. + */ +TCB *tcb_new(void); + + +/* Destroy block, freeing all associated memory with it */ +void tcb_destroy(TCB *block); + + +#endif diff --git a/chapters/compute/scheduling/guides/libult/solution/test_ult.c b/chapters/compute/scheduling/guides/libult/solution/test_ult.c new file mode 100644 index 0000000000..13366e762b --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/test_ult.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include "./threads.h" +#include "utils/utils.h" + +#define NUM_ITER 10000000 +#define NUM_THREADS 2 + +static void *printer_thread(void *arg) +{ + size_t id = (size_t)arg; + int i; + + for (i = 0; i < NUM_ITER; ++i) + if (!(i % 1000000)) + printf("Thread %zu: i = %d\n", id, i); + + return NULL; +} + +int main(void) +{ + size_t i; + int threads[NUM_THREADS]; + void *res; + int rc; + + puts("Hello, this is the _main() func."); + + for (i = 0; i < NUM_THREADS; ++i) { + threads[i] = threads_create(printer_thread, (void *)i); + DIE(threads[i] < 0, "threads_create"); + } + + for (i = 0; i < NUM_THREADS; ++i) + while (1) { + rc = threads_join(threads[i], &res); + DIE(rc < 0, "threads_join"); + + if (rc) { + printf("Joined thread %d with result %p\n", + threads[i], res); + break; + } + } + + return 0; +} diff --git a/chapters/compute/scheduling/guides/libult/solution/threads.c b/chapters/compute/scheduling/guides/libult/solution/threads.c new file mode 100644 index 0000000000..a4f77cba48 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/threads.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +#include "threads.h" +#include "tcb.h" +#include "queue.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static struct queue *ready, *completed; +static TCB *running; + + +static bool init_queues(void); +static bool init_first_context(void); +static bool init_profiling_timer(void); + +static void handle_sigprof(int, siginfo_t *, void *); +static void handle_thread_start(void); + +static bool malloc_stack(TCB *); + +static void block_sigprof(void); +static void unblock_sigprof(void); + + +int threads_create(void *(*start_routine) (void *), void *arg) +{ + block_sigprof(); + + // Init if necessary + + static bool initialized; + + if (!initialized) { + if (!init_queues()) + abort(); + + if (!init_first_context()) + abort(); + + if (!init_profiling_timer()) + abort(); + + initialized = true; + } + + // Create a thread control block for the newly created thread. + + TCB *new = tcb_new(); + + if (new == NULL) + return -1; + + if (getcontext(&new->context) == -1) { + tcb_destroy(new); + return -1; + } + + if (!malloc_stack(new)) { + tcb_destroy(new); + return -1; + } + + makecontext(&new->context, handle_thread_start, 1, new->id); + + new->start_routine = start_routine; + new->argument = arg; + + // Enqueue the newly created stack + + if (queue_enqueue(ready, new) != 0) { + tcb_destroy(new); + return -1; + } + + unblock_sigprof(); + return new->id; +} + + +void threads_exit(void *result) +{ + if (running == NULL) + exit(EXIT_SUCCESS); + + block_sigprof(); + running->return_value = result; + + if (queue_enqueue(completed, running) != 0) + abort(); + + running = queue_dequeue(ready); + if (running == NULL) + exit(EXIT_SUCCESS); + + setcontext(&running->context); // also unblocks SIGPROF +} + + +int threads_join(int id, void **result) +{ + if (id < 0) { + errno = EINVAL; + return -1; + } + + block_sigprof(); + TCB *block = queue_remove_id(completed, id); + + unblock_sigprof(); + + if (block == NULL) + return 0; + + *result = block->return_value; + tcb_destroy(block); + return id; +} + + +static bool init_queues(void) +{ + ready = queue_new(); + if (ready == NULL) + return false; + + completed = queue_new(); + if (completed == NULL) { + queue_destroy(ready); + return false; + } + + return true; +} + +static bool init_first_context(void) +{ + TCB *block = tcb_new(); + + if (block == NULL) + return false; + + if (getcontext(&block->context) == -1) { + tcb_destroy(block); + return false; + } + + running = block; + return true; +} + + +static bool init_profiling_timer(void) +{ + // Install signal handler + + sigset_t all; + + sigfillset(&all); + + const struct sigaction alarm = { + .sa_sigaction = handle_sigprof, + .sa_mask = all, + .sa_flags = SA_SIGINFO | SA_RESTART + }; + + struct sigaction old; + + if (sigaction(SIGPROF, &alarm, &old) == -1) { + perror("sigaction"); + abort(); + } + + const struct itimerval timer = { + { 0, 10000 }, + { 0, 1 } // arms the timer as soon as possible + }; + + // Enable timer + + if (setitimer(ITIMER_PROF, &timer, NULL) == -1) { + if (sigaction(SIGPROF, &old, NULL) == -1) { + perror("sigaction"); + abort(); + } + + return false; + } + + return true; +} + + +static void handle_sigprof(int signum, siginfo_t *nfo, void *context) +{ + int old_errno = errno; + + (void)signum; + (void)nfo; + + if (running == NULL && queue_size(ready) == 0) + _exit(EXIT_SUCCESS); + + // Backup the current context + + ucontext_t *stored = &running->context; + ucontext_t *updated = (ucontext_t *) context; + + stored->uc_flags = updated->uc_flags; + stored->uc_link = updated->uc_link; + stored->uc_mcontext = updated->uc_mcontext; + stored->uc_sigmask = updated->uc_sigmask; + + // Round robin + + if (queue_enqueue(ready, running) != 0) + abort(); + + running = queue_dequeue(ready); + if (running == NULL) + abort(); + + // Manually leave the signal handler + + errno = old_errno; + if (setcontext(&running->context) == -1) + abort(); +} + + +static void handle_thread_start(void) +{ + block_sigprof(); + TCB *this = running; + + unblock_sigprof(); + + void *result = this->start_routine(this->argument); + + threads_exit(result); +} + + +static bool malloc_stack(TCB *thread) +{ + // Get the stack size + + struct rlimit limit; + + if (getrlimit(RLIMIT_STACK, &limit) == -1) + return false; + + // Allocate memory + + void *stack = malloc(limit.rlim_cur); + + if (stack == NULL) + return false; + + // Update the thread control bock + + thread->context.uc_stack.ss_flags = 0; + thread->context.uc_stack.ss_size = limit.rlim_cur; + thread->context.uc_stack.ss_sp = stack; + thread->has_dynamic_stack = true; + + return true; +} + + +static void block_sigprof(void) +{ + sigset_t sigprof; + + sigemptyset(&sigprof); + sigaddset(&sigprof, SIGPROF); + + if (sigprocmask(SIG_BLOCK, &sigprof, NULL) == -1) { + perror("sigprocmask"); + abort(); + } +} + + +static void unblock_sigprof(void) +{ + sigset_t sigprof; + + sigemptyset(&sigprof); + sigaddset(&sigprof, SIGPROF); + + if (sigprocmask(SIG_UNBLOCK, &sigprof, NULL) == -1) { + perror("sigprocmask"); + abort(); + } +} diff --git a/chapters/compute/scheduling/guides/libult/solution/threads.h b/chapters/compute/scheduling/guides/libult/solution/threads.h new file mode 100644 index 0000000000..255a802e62 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/threads.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Interface to a barebones user level thread library. + */ + +#ifndef THREADS_H +#define THREADS_H + + +/* Create a new thread. func is the function that will be run once the + * thread starts execution and arg is the argument for that + * function. On success, returns an id equal or greater to 0. On + * failure, errno is set and -1 is returned. + */ +int threads_create(void *(*start_routine) (void *), void *arg); + + +/* Stop execution of the thread calling this function. */ +void threads_exit(void *result); + + +/* Wait for the thread with matching id to finish execution, that is, + * for it to call threads_exit. On success, the threads result is + * written into result and id is returned. If no completed thread with + * matching id exists, 0 is returned. On error, -1 is returned and + * errno is set. + */ +int threads_join(int id, void **result); + + +#endif diff --git a/chapters/compute/scheduling/guides/libult/solution/utils/log/CPPLINT.cfg b/chapters/compute/scheduling/guides/libult/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/scheduling/guides/libult/solution/utils/log/log.c b/chapters/compute/scheduling/guides/libult/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/scheduling/guides/libult/solution/utils/log/log.h b/chapters/compute/scheduling/guides/libult/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.c b/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.h b/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/scheduling/guides/libult/solution/utils/utils.h b/chapters/compute/scheduling/guides/libult/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/scheduling/guides/libult/support/.gitignore b/chapters/compute/scheduling/guides/libult/support/.gitignore new file mode 100644 index 0000000000..7cb003edb1 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/.gitignore @@ -0,0 +1 @@ +test_ult diff --git a/chapters/compute/scheduling/guides/libult/support/CPPLINT.cfg b/chapters/compute/scheduling/guides/libult/support/CPPLINT.cfg new file mode 100644 index 0000000000..ab07850428 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=(queue|tcb|threads)\.(c|h) diff --git a/chapters/compute/scheduling/guides/libult/support/Makefile b/chapters/compute/scheduling/guides/libult/support/Makefile new file mode 100644 index 0000000000..8b1fb2a9e3 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/Makefile @@ -0,0 +1,47 @@ +TEST = test_ult +LIBULT = libult.so + +all: $(TEST) $(LIBULT) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +LIB_OBJECTS = queue.o tcb.o threads.o +TEST_OBJECTS = test_ult.o $(LOGGER) +LDFLAGS = -L. +LDLIBS = -lult + +$(TEST): $(TEST_OBJECTS) $(LIBULT) + $(CC) -o $@ $(TEST_OBJECTS) $(LDFLAGS) $(LDLIBS) + +$(LIBULT): $(LIB_OBJECTS) + $(CC) -shared -o $@ $^ + +clean:: + rm -f $(TEST) $(LIBULT) + +.PHONY: all clean diff --git a/chapters/compute/scheduling/guides/libult/support/queue.c b/chapters/compute/scheduling/guides/libult/support/queue.c new file mode 100644 index 0000000000..c887b4af72 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/queue.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +#include "queue.h" + + +#include +#include + + +struct node { + TCB *thread; + struct node *next; +}; + + +struct queue { + struct node *head; + size_t size; +}; + +QUEUE *queue_new(void) +{ + QUEUE *new = calloc(1, sizeof(QUEUE)); + + if (new == NULL) + return NULL; + + return new; +} + + +void queue_destroy(QUEUE *queue) +{ + struct node *prev = NULL; + struct node *cursor = queue->head; + + while (cursor != NULL) { + prev = cursor; + cursor = cursor->next; + + tcb_destroy(prev->thread); + free(prev); + } + + free(queue); +} + + +size_t queue_size(const QUEUE *queue) +{ + return queue->size; +} + + +int queue_enqueue(QUEUE *queue, TCB *elem) +{ + // Create the new node + + struct node *new = malloc(sizeof(struct node)); + + if (new == NULL) + return errno; + + new->thread = elem; + new->next = NULL; + + // Enqueue the new node + + if (queue->head == NULL) { + queue->head = new; + } else { + struct node *parent = queue->head; + + while (parent->next != NULL) + parent = parent->next; + parent->next = new; + } + + queue->size += 1; + return 0; +} + + +TCB *queue_dequeue(QUEUE *queue) +{ + struct node *old_head = queue->head; + + if (old_head == NULL || queue->size == 0) + return NULL; + queue->head = queue->head->next; + queue->size -= 1; + + TCB *retval = old_head->thread; + + free(old_head); + + return retval; +} + + +TCB *queue_remove_id(QUEUE *queue, int id) +{ + if (queue->head == NULL) + return NULL; + + struct node *prev = NULL; + struct node *cur = queue->head; + + while (cur != NULL) { + if (cur->thread->id == id) { + if (prev == NULL) + queue->head = cur->next; + else + prev->next = cur->next; + + TCB *retval = cur->thread; + + free(cur); + return retval; + } + + prev = cur; + cur = cur->next; + } + + return NULL; +} diff --git a/chapters/compute/scheduling/guides/libult/support/queue.h b/chapters/compute/scheduling/guides/libult/support/queue.h new file mode 100644 index 0000000000..022d990829 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/queue.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: MIT */ + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Defines a queue to mange TCB elements. + */ + +#ifndef QUEUE_H +#define QUEUE_H + + +#include "tcb.h" + +#include + + +typedef struct queue QUEUE; + + +/* Create a new initialized QUEUE on the heap. Returns a pointer to + * the new block or NULL on error. + */ +QUEUE *queue_new(void); + + +/* Destroy queue, freeing all associated memory with it. It also frees + * all memory of the elements inside the queue. + */ +void queue_destroy(QUEUE *queue); + + +/* Return the number of items in queue. */ +size_t queue_size(const QUEUE *queue); + + +/* Add elem to the end of queue. Returns 0 on succes and non-zero on + * failure. + */ +int queue_enqueue(QUEUE *queue, TCB *elem); + + +/* Remove the first item from the queue and return it. The caller will + * have to free the reuturned element. Returns NULL if the queue is + * empty. + */ +TCB *queue_dequeue(QUEUE *queue); + + +/* Remove element with matching id from the queue. Returns the removed + * element or NULL if no such element exists. + */ +TCB *queue_remove_id(QUEUE *queue, int id); + + +#endif diff --git a/chapters/compute/scheduling/guides/libult/support/tcb.c b/chapters/compute/scheduling/guides/libult/support/tcb.c new file mode 100644 index 0000000000..00d8b26cc7 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/tcb.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +#include "tcb.h" + +#include + + +TCB *tcb_new(void) +{ + static int next_id = 1; + + TCB *new = calloc(1, sizeof(TCB)); + + if (new == NULL) + return NULL; + + new->id = next_id++; + return new; +} + + +void tcb_destroy(TCB *block) +{ + if (block->has_dynamic_stack) + free(block->context.uc_stack.ss_sp); + + free(block); +} diff --git a/chapters/compute/scheduling/guides/libult/support/tcb.h b/chapters/compute/scheduling/guides/libult/support/tcb.h new file mode 100644 index 0000000000..ac2e19f00f --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/tcb.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ + +/** + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Defines the TCB (thread control block) data structure and functions + * to work with it. + */ + +#ifndef TCB_H +#define TCB_H + + +#include +#include +#include + + +typedef struct { + int id; + ucontext_t context; + bool has_dynamic_stack; + void *(*start_routine)(void *arg); + void *argument; + void *return_value; +} TCB; + + +/* Create a new zeroed TCB on the heap. Returns a pointer to the new + * block or NULL on error. + */ +TCB *tcb_new(void); + + +/* Destroy block, freeing all associated memory with it */ +void tcb_destroy(TCB *block); + + +#endif diff --git a/chapters/compute/scheduling/guides/libult/support/test_ult.c b/chapters/compute/scheduling/guides/libult/support/test_ult.c new file mode 100644 index 0000000000..13366e762b --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/test_ult.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include "./threads.h" +#include "utils/utils.h" + +#define NUM_ITER 10000000 +#define NUM_THREADS 2 + +static void *printer_thread(void *arg) +{ + size_t id = (size_t)arg; + int i; + + for (i = 0; i < NUM_ITER; ++i) + if (!(i % 1000000)) + printf("Thread %zu: i = %d\n", id, i); + + return NULL; +} + +int main(void) +{ + size_t i; + int threads[NUM_THREADS]; + void *res; + int rc; + + puts("Hello, this is the _main() func."); + + for (i = 0; i < NUM_THREADS; ++i) { + threads[i] = threads_create(printer_thread, (void *)i); + DIE(threads[i] < 0, "threads_create"); + } + + for (i = 0; i < NUM_THREADS; ++i) + while (1) { + rc = threads_join(threads[i], &res); + DIE(rc < 0, "threads_join"); + + if (rc) { + printf("Joined thread %d with result %p\n", + threads[i], res); + break; + } + } + + return 0; +} diff --git a/chapters/compute/scheduling/guides/libult/support/threads.c b/chapters/compute/scheduling/guides/libult/support/threads.c new file mode 100644 index 0000000000..a4f77cba48 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/threads.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: MIT + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +#include "threads.h" +#include "tcb.h" +#include "queue.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static struct queue *ready, *completed; +static TCB *running; + + +static bool init_queues(void); +static bool init_first_context(void); +static bool init_profiling_timer(void); + +static void handle_sigprof(int, siginfo_t *, void *); +static void handle_thread_start(void); + +static bool malloc_stack(TCB *); + +static void block_sigprof(void); +static void unblock_sigprof(void); + + +int threads_create(void *(*start_routine) (void *), void *arg) +{ + block_sigprof(); + + // Init if necessary + + static bool initialized; + + if (!initialized) { + if (!init_queues()) + abort(); + + if (!init_first_context()) + abort(); + + if (!init_profiling_timer()) + abort(); + + initialized = true; + } + + // Create a thread control block for the newly created thread. + + TCB *new = tcb_new(); + + if (new == NULL) + return -1; + + if (getcontext(&new->context) == -1) { + tcb_destroy(new); + return -1; + } + + if (!malloc_stack(new)) { + tcb_destroy(new); + return -1; + } + + makecontext(&new->context, handle_thread_start, 1, new->id); + + new->start_routine = start_routine; + new->argument = arg; + + // Enqueue the newly created stack + + if (queue_enqueue(ready, new) != 0) { + tcb_destroy(new); + return -1; + } + + unblock_sigprof(); + return new->id; +} + + +void threads_exit(void *result) +{ + if (running == NULL) + exit(EXIT_SUCCESS); + + block_sigprof(); + running->return_value = result; + + if (queue_enqueue(completed, running) != 0) + abort(); + + running = queue_dequeue(ready); + if (running == NULL) + exit(EXIT_SUCCESS); + + setcontext(&running->context); // also unblocks SIGPROF +} + + +int threads_join(int id, void **result) +{ + if (id < 0) { + errno = EINVAL; + return -1; + } + + block_sigprof(); + TCB *block = queue_remove_id(completed, id); + + unblock_sigprof(); + + if (block == NULL) + return 0; + + *result = block->return_value; + tcb_destroy(block); + return id; +} + + +static bool init_queues(void) +{ + ready = queue_new(); + if (ready == NULL) + return false; + + completed = queue_new(); + if (completed == NULL) { + queue_destroy(ready); + return false; + } + + return true; +} + +static bool init_first_context(void) +{ + TCB *block = tcb_new(); + + if (block == NULL) + return false; + + if (getcontext(&block->context) == -1) { + tcb_destroy(block); + return false; + } + + running = block; + return true; +} + + +static bool init_profiling_timer(void) +{ + // Install signal handler + + sigset_t all; + + sigfillset(&all); + + const struct sigaction alarm = { + .sa_sigaction = handle_sigprof, + .sa_mask = all, + .sa_flags = SA_SIGINFO | SA_RESTART + }; + + struct sigaction old; + + if (sigaction(SIGPROF, &alarm, &old) == -1) { + perror("sigaction"); + abort(); + } + + const struct itimerval timer = { + { 0, 10000 }, + { 0, 1 } // arms the timer as soon as possible + }; + + // Enable timer + + if (setitimer(ITIMER_PROF, &timer, NULL) == -1) { + if (sigaction(SIGPROF, &old, NULL) == -1) { + perror("sigaction"); + abort(); + } + + return false; + } + + return true; +} + + +static void handle_sigprof(int signum, siginfo_t *nfo, void *context) +{ + int old_errno = errno; + + (void)signum; + (void)nfo; + + if (running == NULL && queue_size(ready) == 0) + _exit(EXIT_SUCCESS); + + // Backup the current context + + ucontext_t *stored = &running->context; + ucontext_t *updated = (ucontext_t *) context; + + stored->uc_flags = updated->uc_flags; + stored->uc_link = updated->uc_link; + stored->uc_mcontext = updated->uc_mcontext; + stored->uc_sigmask = updated->uc_sigmask; + + // Round robin + + if (queue_enqueue(ready, running) != 0) + abort(); + + running = queue_dequeue(ready); + if (running == NULL) + abort(); + + // Manually leave the signal handler + + errno = old_errno; + if (setcontext(&running->context) == -1) + abort(); +} + + +static void handle_thread_start(void) +{ + block_sigprof(); + TCB *this = running; + + unblock_sigprof(); + + void *result = this->start_routine(this->argument); + + threads_exit(result); +} + + +static bool malloc_stack(TCB *thread) +{ + // Get the stack size + + struct rlimit limit; + + if (getrlimit(RLIMIT_STACK, &limit) == -1) + return false; + + // Allocate memory + + void *stack = malloc(limit.rlim_cur); + + if (stack == NULL) + return false; + + // Update the thread control bock + + thread->context.uc_stack.ss_flags = 0; + thread->context.uc_stack.ss_size = limit.rlim_cur; + thread->context.uc_stack.ss_sp = stack; + thread->has_dynamic_stack = true; + + return true; +} + + +static void block_sigprof(void) +{ + sigset_t sigprof; + + sigemptyset(&sigprof); + sigaddset(&sigprof, SIGPROF); + + if (sigprocmask(SIG_BLOCK, &sigprof, NULL) == -1) { + perror("sigprocmask"); + abort(); + } +} + + +static void unblock_sigprof(void) +{ + sigset_t sigprof; + + sigemptyset(&sigprof); + sigaddset(&sigprof, SIGPROF); + + if (sigprocmask(SIG_UNBLOCK, &sigprof, NULL) == -1) { + perror("sigprocmask"); + abort(); + } +} diff --git a/chapters/compute/scheduling/guides/libult/support/threads.h b/chapters/compute/scheduling/guides/libult/support/threads.h new file mode 100644 index 0000000000..255a802e62 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/threads.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ + +/* + * MIT License + * + * Copyright (c) 2020 Andreas Schärtl and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Github link: https://github.com/kissen/threads */ + +/* + * Interface to a barebones user level thread library. + */ + +#ifndef THREADS_H +#define THREADS_H + + +/* Create a new thread. func is the function that will be run once the + * thread starts execution and arg is the argument for that + * function. On success, returns an id equal or greater to 0. On + * failure, errno is set and -1 is returned. + */ +int threads_create(void *(*start_routine) (void *), void *arg); + + +/* Stop execution of the thread calling this function. */ +void threads_exit(void *result); + + +/* Wait for the thread with matching id to finish execution, that is, + * for it to call threads_exit. On success, the threads result is + * written into result and id is returned. If no completed thread with + * matching id exists, 0 is returned. On error, -1 is returned and + * errno is set. + */ +int threads_join(int id, void **result); + + +#endif diff --git a/chapters/compute/scheduling/guides/libult/support/utils/log/CPPLINT.cfg b/chapters/compute/scheduling/guides/libult/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/scheduling/guides/libult/support/utils/log/log.c b/chapters/compute/scheduling/guides/libult/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/scheduling/guides/libult/support/utils/log/log.h b/chapters/compute/scheduling/guides/libult/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.c b/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.h b/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/scheduling/guides/libult/support/utils/utils.h b/chapters/compute/scheduling/guides/libult/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/scheduling/guides/libult/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lecture/media/context-switch-tcb.svg b/chapters/compute/scheduling/media/context-switch-tcb.svg similarity index 100% rename from content/chapters/compute/lecture/media/context-switch-tcb.svg rename to chapters/compute/scheduling/media/context-switch-tcb.svg diff --git a/content/chapters/compute/lecture/media/priority-round-robin.svg b/chapters/compute/scheduling/media/priority-round-robin.svg similarity index 100% rename from content/chapters/compute/lecture/media/priority-round-robin.svg rename to chapters/compute/scheduling/media/priority-round-robin.svg diff --git a/content/chapters/compute/lecture/media/round-robin/round-robin-0.svg b/chapters/compute/scheduling/media/round-robin/round-robin-0.svg similarity index 100% rename from content/chapters/compute/lecture/media/round-robin/round-robin-0.svg rename to chapters/compute/scheduling/media/round-robin/round-robin-0.svg diff --git a/content/chapters/compute/lecture/media/round-robin/round-robin-1.svg b/chapters/compute/scheduling/media/round-robin/round-robin-1.svg similarity index 100% rename from content/chapters/compute/lecture/media/round-robin/round-robin-1.svg rename to chapters/compute/scheduling/media/round-robin/round-robin-1.svg diff --git a/content/chapters/compute/lecture/media/round-robin/round-robin-2.svg b/chapters/compute/scheduling/media/round-robin/round-robin-2.svg similarity index 100% rename from content/chapters/compute/lecture/media/round-robin/round-robin-2.svg rename to chapters/compute/scheduling/media/round-robin/round-robin-2.svg diff --git a/content/chapters/compute/lecture/media/round-robin/round-robin-3.svg b/chapters/compute/scheduling/media/round-robin/round-robin-3.svg similarity index 100% rename from content/chapters/compute/lecture/media/round-robin/round-robin-3.svg rename to chapters/compute/scheduling/media/round-robin/round-robin-3.svg diff --git a/content/chapters/compute/lecture/media/round-robin/round-robin-4.svg b/chapters/compute/scheduling/media/round-robin/round-robin-4.svg similarity index 100% rename from content/chapters/compute/lecture/media/round-robin/round-robin-4.svg rename to chapters/compute/scheduling/media/round-robin/round-robin-4.svg diff --git a/content/chapters/compute/lecture/media/round-robin/round-robin-5.svg b/chapters/compute/scheduling/media/round-robin/round-robin-5.svg similarity index 100% rename from content/chapters/compute/lecture/media/round-robin/round-robin-5.svg rename to chapters/compute/scheduling/media/round-robin/round-robin-5.svg diff --git a/content/chapters/compute/lecture/media/round-robin/round-robin-6.svg b/chapters/compute/scheduling/media/round-robin/round-robin-6.svg similarity index 100% rename from content/chapters/compute/lecture/media/round-robin/round-robin-6.svg rename to chapters/compute/scheduling/media/round-robin/round-robin-6.svg diff --git a/content/chapters/compute/lecture/media/round-robin/round-robin-7.svg b/chapters/compute/scheduling/media/round-robin/round-robin-7.svg similarity index 100% rename from content/chapters/compute/lecture/media/round-robin/round-robin-7.svg rename to chapters/compute/scheduling/media/round-robin/round-robin-7.svg diff --git a/content/chapters/compute/lecture/media/thread-behaviour.svg b/chapters/compute/scheduling/media/thread-behaviour.svg similarity index 100% rename from content/chapters/compute/lecture/media/thread-behaviour.svg rename to chapters/compute/scheduling/media/thread-behaviour.svg diff --git a/content/chapters/compute/lecture/media/thread-states-extended.svg b/chapters/compute/scheduling/media/thread-states-extended.svg similarity index 100% rename from content/chapters/compute/lecture/media/thread-states-extended.svg rename to chapters/compute/scheduling/media/thread-states-extended.svg diff --git a/content/chapters/compute/lab/content/scheduling.md b/chapters/compute/scheduling/reading/scheduling.md similarity index 68% rename from content/chapters/compute/lab/content/scheduling.md rename to chapters/compute/scheduling/reading/scheduling.md index af275eba3f..3bfed83c2f 100644 --- a/content/chapters/compute/lab/content/scheduling.md +++ b/chapters/compute/scheduling/reading/scheduling.md @@ -32,33 +32,6 @@ So if we cannot run code in parallel with ULTs, then why use them? Well, programs that create many context switches may suffer from the larger overhead if they use kernel-level threads. In such cases, user-level threads may be useful as context switches bring less overhead between user-level threads. -## Practice: User-Level Threads Scheduler - -Go to `support/libult`. -It contains a minimalist **user-level threads** scheduler. -Compiling it produces a shared library called `libult.so`. -You can also consult its [documentation](https://www.schaertl.me/posts/a-bare-bones-user-level-thread-library/). -It explains the API as well as its implementation. -The API exposed by the scheduling library is very simple. -It is only made up of 3 functions: - -- `threads_create()` creates a new ULT -- `threads_exit()` moves the current ULT to the COMPLETED state -- `threads_join()` waits for a given thread to end and saves its return value in the `result` argument - -Look inside `support/libult/threads.c`. -Here you will find the 3 functions mentioned above. - -The scheduler only uses 3 states: RUNNING, READY, COMPLETED. - -[Quiz](../quiz/number-of-running-ults.md) - -The threads in the READY and COMPLETED states are kept in 2 separate queues. -When the scheduler wants to run a new thread, it retrieves it from the READY queue. -When a thread ends its execution, it is added to the COMPLETED queue, together with its return value. - -[Quiz](../quiz/why-use-completed-queue.md) - ## Thread Control Block Let's dissect the `threads_create()` function a bit. @@ -67,39 +40,39 @@ We'll discuss preemption [in the next section](#scheduling---how-is-it-done). After performing initialisations, the function creates a `TCB` object. TCB stands for **Thread Control Block**. -During the [lecture](../../lecture/), you saw that the kernel stores one instance of a [`task_struct`](https://elixir.bootlin.com/linux/v5.19.11/source/include/linux/sched.h#L726) for each thread. +During the lecture, you saw that the kernel stores one instance of a [`task_struct`](https://elixir.bootlin.com/linux/v5.19.11/source/include/linux/sched.h#L726) for each thread. Remember that its most important fields are: ```C struct task_struct { - unsigned int __state; + unsigned int __state; - void *stack; + void *stack; - unsigned int flags; + unsigned int flags; - int on_cpu; - int prio; + int on_cpu; + int prio; - /* Scheduler information */ - struct sched_entity se; - const struct sched_class *sched_class; + /* Scheduler information */ + struct sched_entity se; + const struct sched_class *sched_class; - /* The VAS: memory mappings */ - struct mm_struct *mm; + /* The VAS: memory mappings */ + struct mm_struct *mm; - int exit_state; - int exit_code; + int exit_state; + int exit_code; - pid_t pid; + pid_t pid; - struct task_struct __rcu *parent; + struct task_struct __rcu *parent; - /* Child processes */ - struct list_head children; + /* Child processes */ + struct list_head children; - /* Open file information */ - struct files_struct *files; + /* Open file information */ + struct files_struct *files; }; ``` @@ -108,12 +81,12 @@ The same is true about the `TCB` in `libult.so`: ```c typedef struct { - int id; - ucontext_t context; - bool has_dynamic_stack; - void *(*start_routine) (void *); - void *argument; - void *return_value; + int id; + ucontext_t context; + bool has_dynamic_stack; + void *(*start_routine) (void *); + void *argument; + void *return_value; } TCB; ``` @@ -131,25 +104,25 @@ We'll look at the [`uk_thread`](https://github.com/unikraft/unikraft/blob/9bf6e6 ```c struct uk_thread { - const char *name; - void *stack; - void *tls; - void *ctx; - UK_TAILQ_ENTRY(struct uk_thread) thread_list; - uint32_t flags; - __snsec wakeup_time; - bool detached; - struct uk_waitq waiting_threads; - struct uk_sched *sched; - void (*entry)(void *); - void *arg; - void *prv; + const char *name; + void *stack; + void *tls; + void *ctx; + UK_TAILQ_ENTRY(struct uk_thread) thread_list; + uint32_t flags; + __snsec wakeup_time; + bool detached; + struct uk_waitq waiting_threads; + struct uk_sched *sched; + void (*entry)(void *); + void *arg; + void *prv; }; ``` There are some visible similarities between the two TCBs. -[Quiz](../quiz/tcb-libult-unikraft.md) +[Quiz](questions/tcb-libult-unikraft.md) Therefore, the workflow for creating and running a thread goes like this: @@ -158,17 +131,17 @@ main thread | `--> threads_create() | - |--> tcb_new() + |--> tcb_new() `--> makecontext() - | - `--> handle_thread_start() - called using the context - | - |--> start_routine() - the thread runs + | + `--> handle_thread_start() - called using the context + | + |--> start_routine() - the thread runs `--> threads_exit() ``` -Compile and run the code in `support/libult/test_ult.c`. -If you encounter the following error when running `test_ult`, remember what you learned about the loader and using custom shared libraries in the [Software Stack lab](../../../software-stack/lab). +Compile and run the code in `libult/support/test_ult.c`. +If you encounter the following error when running `test_ult`, remember what you learned about the loader and using custom shared libraries in the [Software Stack lab](reading/libc.md). ```console ./test_ult: error while loading shared libraries: libult.so: cannot open shared object file: No such file or directory @@ -179,7 +152,7 @@ If you encounter the following error when running `test_ult`, remember what you Notice that the threads run their code and alternatively, because their prints appear interleaved. [In the next section](#scheduling---how-is-it-done), we'll see how this is done. -[Quiz](../quiz/ult-thread-ids.md) +[Quiz](questions/ult-thread-ids.md) ## Scheduling - How is it done? @@ -192,7 +165,7 @@ So, yielding the CPU triggers a context switch whereby the current thread stops Cooperative scheduling relies on the fact that threads themselves would yield the CPU at some point. If threads don't abide by this convention, they end up monopolising the CPU (since there is no one to suspend them) and end up starving the others. -You can get a feel of this behaviour by running the cooperative [scheduler from Unikraft](https://github.com/unikraft/unikraft/blob/staging/lib/ukschedcoop/schedcoop.c) in the [lecture demos](../../lecture/demo/cooperative-scheduling). +You can get a feel of this behaviour by running the cooperative [scheduler from Unikraft](https://github.com/unikraft/unikraft/blob/staging/lib/ukschedcoop/schedcoop.c) in the [lecture demos]. This type of schedulers have the advantage of being lightweight, thus resulting in less overhead caused by context switches. However, as we've already stated, they rely on the "good will" of threads to yield the CPU at some point. @@ -206,12 +179,12 @@ Preemptive schedulers allow threads to run only for a maximum amount of time, ca They use timers which fire when a new time slice passes. The firing of one such timer causes a context switch whereby the currently RUNNING thread is _preempted_ (i.e. suspended) and replaced with another one. -[Quiz](../quiz/type-of-scheduler-in-libult.md) +[Quiz](questions/type-of-scheduler-in-libult.md) Look at the `init_profiling_timer()` function. It creates a timer that generates a `SIGPROF` signal and then defines a handler (the `handle_sigprof()` function) that is executed whenever the `SIGPROF` signal is received. -[Quiz](../quiz/time-slice-value.md) +[Quiz](questions/time-slice-value.md) It is this handler that performs the context switch per se. Look at its code. @@ -232,11 +205,11 @@ This algorithm (that schedules the first thread in the READY queue) is called _R ```C if (queue_enqueue(ready, running) != 0) { - abort(); + abort(); } if ((running = queue_dequeue(ready)) == NULL) { - abort(); + abort(); } ``` @@ -244,18 +217,8 @@ The new `running` thread is resumed upon setting the current context to it: ```C if (setcontext(&running->context) == -1) { - abort(); + abort(); } ``` This is how scheduling is done! - -### Practice: Another Time Slice - -1. Modify the time slice set to the timer to 2 seconds. -Re-run the code in `support/libult/test_ult.c`. -Notice that now no context switch happens between the 2 created threads because they end before the timer can fire. - -1. Now change the `printer_thread()` function in `test_ult.c` to make it run for more than 2 seconds. -See that now the prints from the two threads appear intermingled. -Add prints to the `handle_sigprof()` function in `support/libult/threads.c` to see the context switch happen. diff --git a/content/chapters/compute/lecture/slides/scheduling-algorithms.md b/chapters/compute/scheduling/slides/scheduling-algorithms.md similarity index 93% rename from content/chapters/compute/lecture/slides/scheduling-algorithms.md rename to chapters/compute/scheduling/slides/scheduling-algorithms.md index 0102488bf7..dae3c90243 100644 --- a/content/chapters/compute/lecture/slides/scheduling-algorithms.md +++ b/chapters/compute/scheduling/slides/scheduling-algorithms.md @@ -36,7 +36,7 @@ * **Round-Robin**: * Push each new thread to a queue - * Every time slice, dequeue one thread, run it, then reenqueue it + * Every time slice, dequeue one thread, run it, then requeue it * Threads run in the order in which they're spawned ---- @@ -74,7 +74,7 @@ #### Round-Robin - Live Scheduling -![Round-Robin Scheduling](./media/round-robin-generated.gif) +![Round-Robin Scheduling](../generated-media/round-robin-generated.gif) ---- @@ -96,8 +96,8 @@ ```c struct schedcoop_private { - struct uk_thread_list thread_list; - struct uk_thread_list sleeping_threads; + struct uk_thread_list thread_list; + struct uk_thread_list sleeping_threads; }; ``` @@ -106,7 +106,7 @@ struct schedcoop_private { ```c static void schedcoop_yield(struct uk_sched *s) { - schedcoop_schedule(s); + schedcoop_schedule(s); } ``` @@ -205,7 +205,7 @@ student@os:~$ ps -e -o pid,ni,comm | grep 5753 * The scheduler keeps a list of queues, one for each priority * During a context switch, the new RUNNING thread is chosen from the highest priority non-empty queue -![Priority-Based Round-Robin](media/priority-round-robin.svg) +![Priority-Based Round-Robin](../media/priority-round-robin.svg) --- diff --git a/content/chapters/compute/lecture/slides/scheduling.md b/chapters/compute/scheduling/slides/scheduling.md similarity index 84% rename from content/chapters/compute/lecture/slides/scheduling.md rename to chapters/compute/scheduling/slides/scheduling.md index d9721907c1..a8e14b9cf6 100644 --- a/content/chapters/compute/lecture/slides/scheduling.md +++ b/chapters/compute/scheduling/slides/scheduling.md @@ -14,7 +14,7 @@ #### Typical Thread Execution -![Thread Execution](media/thread-behaviour.svg) +![Thread Execution](../media/thread-behaviour.svg) ---- @@ -22,7 +22,7 @@ * At most one RUNNING thread per core * Queues for BLOCKED and READY threads -* [Quiz](../quiz/number-of-running-threads.md) +* [Quiz](../drills/questions/number-of-running-threads.md) Thread State Diagram @@ -33,7 +33,7 @@ A more detailed diagram is [here](./cool-extra-stuff.md#the-suspended-states) ### Context Switch - When? * Voluntary - initiated by the running thread: - * RUNNING thread performs a blocking action (eg: I/O call) + * RUNNING thread performs a blocking action (e.g.: I/O call) * The thread calls `yield()` / `SwitchToThread()` * Involuntary - initiated by the OS: * RUNNING thread ends @@ -44,7 +44,7 @@ A more detailed diagram is [here](./cool-extra-stuff.md#the-suspended-states) #### Context Switch - How? -![Context Switch and TCBs](media/context-switch-tcb.svg) +![Context Switch and TCBs](../media/context-switch-tcb.svg) --- @@ -65,7 +65,7 @@ A more detailed diagram is [here](./cool-extra-stuff.md#the-suspended-states) * `demo/context-switch/cpu_bound.c`: -```[1 - 4] +```console student@os:~$ cat /proc/$(pidof cpu_bound)/status [...] voluntary_ctxt_switches: 13 @@ -91,6 +91,6 @@ nonvoluntary_ctxt_switches: 3 * Causes of overhead: * running the scheduler itself - * flushing the [TLB](../../data/lecture) of the evicted thread is required + * flushing the [TLB](../../data) of the evicted thread is required * the new thread starts with an empty TLB * Faster context switches between threads of the same process because their TLB is not flushed diff --git a/chapters/compute/slides.mdpp b/chapters/compute/slides.mdpp new file mode 100644 index 0000000000..1ce2620a1b --- /dev/null +++ b/chapters/compute/slides.mdpp @@ -0,0 +1,44 @@ +--- +title: "OS: Compute" +revealOptions: + background-color: 'aquamarine' + transition: 'none' + slideNumber: true + autoAnimateDuration: 0.0 +--- + +# COMPUTE + +1. [Processes](#1-processes) +1. [Threads](#2-threads) +1. [Scheduling](#3-scheduling) +1. [Synchronization](#4-synchronization) + + +!INCLUDE "overview/slides/intro.md" + +!INCLUDE "processes/slides/processes.md" + +!INCLUDE "threads/slides/threads.md" + +!INCLUDE "processes/slides/process-attributes.md" + +!INCLUDE "processes/slides/fork.md" + +!INCLUDE "copy-on-write/slides/copy-on-write.md" + +!INCLUDE "scheduling/slides/scheduling.md" + +!INCLUDE "scheduling/slides/scheduling-algorithms.md" + +!INCLUDE "synchronization/slides/synchronization.md" + +!INCLUDE "synchronization/slides/mutual-exclusion.md" + +!INCLUDE "synchronization/slides/notifications.md" + +!INCLUDE "synchronization/slides/barrier.md" + +!INCLUDE "synchronization/slides/thread-safety.md" + +!INCLUDE "arena/slides/cool-extra-stuff.md" diff --git a/content/chapters/compute/lab/quiz/coarse-vs-granular-critical-section.md b/chapters/compute/synchronization/drills/questions/coarse-vs-granular-critical-section.md similarity index 100% rename from content/chapters/compute/lab/quiz/coarse-vs-granular-critical-section.md rename to chapters/compute/synchronization/drills/questions/coarse-vs-granular-critical-section.md diff --git a/content/chapters/compute/lecture/quiz/not-race-condition.md b/chapters/compute/synchronization/drills/questions/not-race-condition.md similarity index 100% rename from content/chapters/compute/lecture/quiz/not-race-condition.md rename to chapters/compute/synchronization/drills/questions/not-race-condition.md diff --git a/content/chapters/compute/lab/quiz/notify-only-with-mutex.md b/chapters/compute/synchronization/drills/questions/notify-only-with-mutex.md similarity index 100% rename from content/chapters/compute/lab/quiz/notify-only-with-mutex.md rename to chapters/compute/synchronization/drills/questions/notify-only-with-mutex.md diff --git a/content/chapters/compute/lab/quiz/semaphore-equivalent.md b/chapters/compute/synchronization/drills/questions/semaphore-equivalent.md similarity index 100% rename from content/chapters/compute/lab/quiz/semaphore-equivalent.md rename to chapters/compute/synchronization/drills/questions/semaphore-equivalent.md diff --git a/content/chapters/compute/lab/quiz/tls-synchronization.md b/chapters/compute/synchronization/drills/questions/tls-synchronization.md similarity index 100% rename from content/chapters/compute/lab/quiz/tls-synchronization.md rename to chapters/compute/synchronization/drills/questions/tls-synchronization.md diff --git a/content/chapters/compute/lab/quiz/tls-var-copies.md b/chapters/compute/synchronization/drills/questions/tls-var-copies.md similarity index 100% rename from content/chapters/compute/lab/quiz/tls-var-copies.md rename to chapters/compute/synchronization/drills/questions/tls-var-copies.md diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/README.md b/chapters/compute/synchronization/drills/tasks/race-condition/README.md new file mode 100644 index 0000000000..233713fc62 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/README.md @@ -0,0 +1,47 @@ +# Race Conditions + +## C - TLS on Demand + +The perspective of C towards TLS is the following: everything is shared by default. +This makes multithreading easier and more lightweight to implement than in other languages, like D, because synchronization is left entirely up to the developer, at the cost of potential unsafety. + +Of course, we can specify that some data belongs to the TLS, by preceding the declaration of a variable with `__thread` keyword. +First, compile and run the code in `race-condition/support/c/race_condition_tls.c` a few times. +As expected, the result is different each time. + +1. Modify the declaration of `var` and add the `__thread` keyword to place the variable in the TLS of each thread. + Recompile and run the code a few more times. + You should see that in the end, `var` is 0. + + [Quiz 1](../../questions/tls-synchronization.md) + + [Quiz 2](../../questions/tls-var-copies.md) + +1. Print the address and value of `var` in each thread. + See that they differ. + +1. Modify the value of `var` in the `main()` function before calling `pthread_create()`. + Notice that the value doesn't propagate to the other threads. + This is because, upon creating a new thread, its TLS is initialised. + +## Atomic Assembly + +No, this section is not about nukes, sadly :(. +Instead, we aim to get accustomed to the way in which the x86 ISA provides atomic instructions. + +This mechanism looks very simple. +It is but **one instruction prefix**: `lock`. +It is not an instruction with its own separate opcode, but a prefix that slightly modifies the opcode of the following instructions to make the CPU execute it atomically (i.e. with exclusive access to the data bus). + +`lock` must only be place before an instruction that executes a _read-modify-write_ action. +For example, we cannot place it before a `mov` instruction, as the action of a `mov` is simply `read` or `write`. +Instead, we can place it in front of an `inc` instruction if its operand is memory. + +Look at the code in `race-condition/support/asm/race_condition_lock.S`. +It's an Assembly equivalent of the code you've already seen many times so far (such as `race-condition/support/c/race_condition.c`). +Assemble and run it a few times. +Notice the different results you get. + +Now add the `lock` prefix before `inc` and `dec`. +Reassemble and rerun the code. +And now we have synchronized the two threads by leveraging CPU support. diff --git a/content/chapters/compute/lab/solution/race-condition/asm/.gitignore b/chapters/compute/synchronization/drills/tasks/race-condition/solution/asm/.gitignore similarity index 100% rename from content/chapters/compute/lab/solution/race-condition/asm/.gitignore rename to chapters/compute/synchronization/drills/tasks/race-condition/solution/asm/.gitignore diff --git a/content/chapters/compute/lab/support/race-condition/asm/Makefile b/chapters/compute/synchronization/drills/tasks/race-condition/solution/asm/Makefile similarity index 100% rename from content/chapters/compute/lab/support/race-condition/asm/Makefile rename to chapters/compute/synchronization/drills/tasks/race-condition/solution/asm/Makefile diff --git a/content/chapters/compute/lab/solution/race-condition/asm/race_condition_lock.S b/chapters/compute/synchronization/drills/tasks/race-condition/solution/asm/race_condition_lock.S similarity index 96% rename from content/chapters/compute/lab/solution/race-condition/asm/race_condition_lock.S rename to chapters/compute/synchronization/drills/tasks/race-condition/solution/asm/race_condition_lock.S index 4ab900d774..8e64770838 100644 --- a/content/chapters/compute/lab/solution/race-condition/asm/race_condition_lock.S +++ b/chapters/compute/synchronization/drills/tasks/race-condition/solution/asm/race_condition_lock.S @@ -1,4 +1,4 @@ -; SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause */ extern printf extern pthread_create diff --git a/content/chapters/compute/lab/solution/race-condition/c/.gitignore b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/.gitignore similarity index 100% rename from content/chapters/compute/lab/solution/race-condition/c/.gitignore rename to chapters/compute/synchronization/drills/tasks/race-condition/solution/c/.gitignore diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/Makefile b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/Makefile new file mode 100644 index 0000000000..4617bee7c7 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/Makefile @@ -0,0 +1,37 @@ +BINARIES = race_condition race_condition_tls race_condition_mutex race_condition_atomic race_condition_atomic2 +LDLIBS = -lpthread +all: $(BINARIES) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARIES): $(LOGGER) + +clean:: + -rm -f $(BINARIES) + +.PHONY: all clean diff --git a/content/chapters/compute/lab/solution/race-condition/c/race_condition_tls.c b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/race_condition_tls.c similarity index 96% rename from content/chapters/compute/lab/solution/race-condition/c/race_condition_tls.c rename to chapters/compute/synchronization/drills/tasks/race-condition/solution/c/race_condition_tls.c index 1794db7e65..710c189e8c 100644 --- a/content/chapters/compute/lab/solution/race-condition/c/race_condition_tls.c +++ b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/race_condition_tls.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/log/CPPLINT.cfg b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/log/log.c b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/log/log.h b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.c b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.h b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/utils.h b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/solution/race-condition/d/.gitignore b/chapters/compute/synchronization/drills/tasks/race-condition/solution/d/.gitignore similarity index 100% rename from content/chapters/compute/lab/solution/race-condition/d/.gitignore rename to chapters/compute/synchronization/drills/tasks/race-condition/solution/d/.gitignore diff --git a/content/chapters/compute/lab/solution/race-condition/d/Makefile b/chapters/compute/synchronization/drills/tasks/race-condition/solution/d/Makefile similarity index 100% rename from content/chapters/compute/lab/solution/race-condition/d/Makefile rename to chapters/compute/synchronization/drills/tasks/race-condition/solution/d/Makefile diff --git a/content/chapters/compute/lab/solution/race-condition/d/race_condition_atomic.d b/chapters/compute/synchronization/drills/tasks/race-condition/solution/d/race_condition_atomic.d similarity index 100% rename from content/chapters/compute/lab/solution/race-condition/d/race_condition_atomic.d rename to chapters/compute/synchronization/drills/tasks/race-condition/solution/d/race_condition_atomic.d diff --git a/content/chapters/compute/lab/solution/race-condition/d/race_condition_mutex_coarse.d b/chapters/compute/synchronization/drills/tasks/race-condition/solution/d/race_condition_mutex_coarse.d similarity index 86% rename from content/chapters/compute/lab/solution/race-condition/d/race_condition_mutex_coarse.d rename to chapters/compute/synchronization/drills/tasks/race-condition/solution/d/race_condition_mutex_coarse.d index dc410ece08..a6ef5c527d 100644 --- a/content/chapters/compute/lab/solution/race-condition/d/race_condition_mutex_coarse.d +++ b/chapters/compute/synchronization/drills/tasks/race-condition/solution/d/race_condition_mutex_coarse.d @@ -7,30 +7,30 @@ import core.sync.mutex; enum NUM_ITER = 10_000_000; __gshared int var; -shared Mutex mutex; +shared Mutex lock; void incrementVar() { // TODO: wrap the whole `for` statement in the critical section and measure // the running times. - mutex.lock(); + lock.lock(); for (size_t i = 0; i < NUM_ITER; i++) { var++; } - mutex.unlock(); + lock.unlock(); } void decrementVar() { // TODO: wrap the whole `for` statement in the critical section and measure // the running times. - mutex.lock(); + lock.lock(); for (size_t i = 0; i < NUM_ITER; i++) { var--; } - mutex.unlock(); + lock.unlock(); } void main() @@ -39,7 +39,7 @@ void main() import std.concurrency : spawn; import std.stdio : writeln; - mutex = new shared Mutex(); + lock = new shared Mutex(); spawn(&incrementVar); spawn(&decrementVar); diff --git a/content/chapters/compute/lab/support/race-condition/asm/.gitignore b/chapters/compute/synchronization/drills/tasks/race-condition/support/asm/.gitignore similarity index 100% rename from content/chapters/compute/lab/support/race-condition/asm/.gitignore rename to chapters/compute/synchronization/drills/tasks/race-condition/support/asm/.gitignore diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/support/asm/Makefile b/chapters/compute/synchronization/drills/tasks/race-condition/support/asm/Makefile new file mode 100644 index 0000000000..87d70b3c97 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/support/asm/Makefile @@ -0,0 +1,17 @@ +NASM = nasm +ASFLAGS = -f elf64 +CC = gcc +LDFLAGS = -no-pie +LDLIBS = -lpthread +BINARY = race_condition_lock + +.PHONY: all clean + +all: $(BINARY) + +$(BINARY).o: $(BINARY).S + $(NASM) $(ASFLAGS) -o $@ $< + +clean: + -rm -f $(BINARY) $(BINARY).o + -rm -f *~ diff --git a/content/chapters/compute/lab/support/race-condition/asm/race_condition_lock.S b/chapters/compute/synchronization/drills/tasks/race-condition/support/asm/race_condition_lock.S similarity index 96% rename from content/chapters/compute/lab/support/race-condition/asm/race_condition_lock.S rename to chapters/compute/synchronization/drills/tasks/race-condition/support/asm/race_condition_lock.S index 72f1f6b265..0fcc2527e9 100644 --- a/content/chapters/compute/lab/support/race-condition/asm/race_condition_lock.S +++ b/chapters/compute/synchronization/drills/tasks/race-condition/support/asm/race_condition_lock.S @@ -1,4 +1,4 @@ -; SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause */ extern printf extern pthread_create diff --git a/content/chapters/compute/lab/support/race-condition/c/.gitignore b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/.gitignore similarity index 100% rename from content/chapters/compute/lab/support/race-condition/c/.gitignore rename to chapters/compute/synchronization/drills/tasks/race-condition/support/c/.gitignore diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/Makefile b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/Makefile new file mode 100644 index 0000000000..4617bee7c7 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/Makefile @@ -0,0 +1,37 @@ +BINARIES = race_condition race_condition_tls race_condition_mutex race_condition_atomic race_condition_atomic2 +LDLIBS = -lpthread +all: $(BINARIES) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARIES): $(LOGGER) + +clean:: + -rm -f $(BINARIES) + +.PHONY: all clean diff --git a/content/chapters/compute/lab/support/race-condition/c/race_condition.c b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/race_condition.c similarity index 100% rename from content/chapters/compute/lab/support/race-condition/c/race_condition.c rename to chapters/compute/synchronization/drills/tasks/race-condition/support/c/race_condition.c diff --git a/content/chapters/compute/lab/support/race-condition/c/race_condition_atomic.c b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/race_condition_atomic.c similarity index 100% rename from content/chapters/compute/lab/support/race-condition/c/race_condition_atomic.c rename to chapters/compute/synchronization/drills/tasks/race-condition/support/c/race_condition_atomic.c diff --git a/content/chapters/compute/lab/support/race-condition/c/race_condition_atomic2.c b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/race_condition_atomic2.c similarity index 100% rename from content/chapters/compute/lab/support/race-condition/c/race_condition_atomic2.c rename to chapters/compute/synchronization/drills/tasks/race-condition/support/c/race_condition_atomic2.c diff --git a/content/chapters/compute/lab/support/race-condition/c/race_condition_mutex.c b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/race_condition_mutex.c similarity index 100% rename from content/chapters/compute/lab/support/race-condition/c/race_condition_mutex.c rename to chapters/compute/synchronization/drills/tasks/race-condition/support/c/race_condition_mutex.c diff --git a/content/chapters/compute/lab/support/race-condition/c/race_condition_tls.c b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/race_condition_tls.c similarity index 100% rename from content/chapters/compute/lab/support/race-condition/c/race_condition_tls.c rename to chapters/compute/synchronization/drills/tasks/race-condition/support/c/race_condition_tls.c diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/log/CPPLINT.cfg b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/log/log.c b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/log/log.h b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.c b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.h b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/utils.h b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/support/race-condition/python/race_condition.py b/chapters/compute/synchronization/drills/tasks/race-condition/support/python/race_condition.py similarity index 100% rename from content/chapters/compute/lab/support/race-condition/python/race_condition.py rename to chapters/compute/synchronization/drills/tasks/race-condition/support/python/race_condition.py diff --git a/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/README.md b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/README.md new file mode 100644 index 0000000000..2983fea3e6 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/README.md @@ -0,0 +1,11 @@ +# Synchronization - Thread-Safe Data Structure + +Now it's time for a fully practical exercise. +Go to `CLIST/support/`. +In the file `clist.c` you'll find a simple implementation of an array list. +Although correct, it is not (yet) thread-safe. + +The code in `test.c` verifies its single-threaded correctness, while the one in `test_parallel.c` verifies it works properly with multiple threads. +Your task is to synchronize this data structure using whichever primitives you like. +Try to keep the implementation efficient. +Aim to decrease your running times as much as you can. diff --git a/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/.gitignore b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/.gitignore new file mode 100644 index 0000000000..fac5e641d9 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/.gitignore @@ -0,0 +1,2 @@ +test +test_parallel diff --git a/content/chapters/compute/lab/support/CLIST/.jscpd.json b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/.jscpd.json similarity index 100% rename from content/chapters/compute/lab/support/CLIST/.jscpd.json rename to chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/.jscpd.json diff --git a/content/chapters/compute/lab/support/CLIST/CPPLINT.cfg b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/CPPLINT.cfg similarity index 100% rename from content/chapters/compute/lab/support/CLIST/CPPLINT.cfg rename to chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/CPPLINT.cfg diff --git a/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/Makefile b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/Makefile new file mode 100644 index 0000000000..627e8fb2ef --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/Makefile @@ -0,0 +1,50 @@ +TARGET = test +TARGET_PARALLEL = test_parallel + +all: $(TARGET) $(TARGET_PARALLEL) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +LIB_OBJECTS = clist.o +LIBRARY = libclist.a +TEST_OBJECTS = test.o $(LIBRARY) +TEST_OBJECTS_PARALLEL = test_parallel.o $(LIBRARY) + +LDLIBS = -lpthread + +$(LIBRARY): $(LIB_OBJECTS) + ar rcs $(LIBRARY) $(LIB_OBJECTS) + +$(TARGET): $(TEST_OBJECTS) + $(CC) $^ -o $@ $(LDLIBS) + +$(TARGET_PARALLEL): $(TEST_OBJECTS_PARALLEL) + $(CC) $^ -o $@ $(LDLIBS) + +clean:: + -rm -f $(LIBRARY) $(TARGET) $(TARGET_PARALLEL) diff --git a/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/clist.c b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/clist.c new file mode 100644 index 0000000000..3145a6ced4 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/clist.c @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: BSD-2-Clause + +/******************************************/ +/* */ +/* Alexander Agdgomlishvili */ +/* */ +/* cdevelopment@mail.com */ +/* */ +/******************************************/ + +/** + * BSD 2-Clause License + * + * Copyright (c) 2020, Alexander Agdgomlishvili + * cdevelopment@mail.com + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* Github link: https://github.com/AlexanderAgd/CLIST */ + +#include +#include +#include +#include +#include +#include "clist.h" + +typedef struct { + int count; /* Number of items in the list. */ + int alloc_size; /* Allocated size in quantity of items */ + int lastSearchPos; /* Position of last search - firstMatch or LastMatch */ + size_t item_size; /* Size of each item in bytes. */ + void *items; /* Pointer to the list */ +} CList_priv_; + +int CList_Realloc_(CList *l, int n) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + if (n < p->count) { + fprintf(stderr, "CList: ERROR! Can not realloc to '%i' size - count is '%i'\n", n, p->count); + assert(n >= p->count); + return 0; + } + + if (n == 0 && p->alloc_size == 0) + n = 2; + + void *ptr = realloc(p->items, p->item_size * n); + + if (ptr == NULL) { + fprintf(stderr, "CList: ERROR! Can not reallocate memory!\n"); + return 0; + } + p->items = ptr; + p->alloc_size = n; + return 1; +} + +void *CList_Add_(CList *l, void *o) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + if (p->count == p->alloc_size && + CList_Realloc_(l, p->alloc_size * 2) == 0) + return NULL; + + char *data = (char *) p->items; + + data = data + p->count * p->item_size; + memcpy(data, o, p->item_size); + p->count++; + return data; +} + +void *CList_Insert_(CList *l, void *o, int n) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + if (n < 0 || n > p->count) { + fprintf(stderr, "CList: ERROR! Insert position outside range - %d; n - %d.\n", + p->count, n); + assert(n >= 0 && n <= p->count); + return NULL; + } + + if (p->count == p->alloc_size && + CList_Realloc_(l, p->alloc_size * 2) == 0) + return NULL; + + size_t step = p->item_size; + char *data = (char *) p->items + n * step; + + memmove(data + step, data, (p->count - n) * step); + memcpy(data, o, step); + p->count++; + return data; +} + +void *CList_Replace_(CList *l, void *o, int n) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + if (n < 0 || n >= p->count) { + fprintf(stderr, "CList: ERROR! Replace position outside range - %d; n - %d.\n", + p->count, n); + assert(n >= 0 && n < p->count); + return NULL; + } + + char *data = (char *) p->items; + + data = data + n * p->item_size; + memcpy(data, o, p->item_size); + return data; +} + +void CList_Remove_(CList *l, int n) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + if (n < 0 || n >= p->count) { + fprintf(stderr, "CList: ERROR! Remove position outside range - %d; n - %d.\n", + p->count, n); + assert(n >= 0 && n < p->count); + return; + } + + size_t step = p->item_size; + char *data = (char *)p->items + n * step; + + memmove(data, data + step, (p->count - n - 1) * step); + p->count--; + + if (p->alloc_size > 3 * p->count && p->alloc_size >= 4) /* Dont hold much memory */ + CList_Realloc_(l, p->alloc_size / 2); +} + +void *CList_At_(CList *l, int n) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + if (n < 0 || n >= p->count) { + fprintf(stderr, "CList: ERROR! Get position outside range - %d; n - %d.\n", + p->count, n); + assert(n >= 0 && n < p->count); + return NULL; + } + + char *data = (char *) p->items; + + data = data + n * p->item_size; + return data; +} + +void *CList_firstMatch_(CList *l, const void *o, size_t shift, size_t size, int string) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + char *data = (char *) p->items; + size_t step = p->item_size; + + p->lastSearchPos = -1; + + if (shift + size > p->item_size) { + char *errmsg = "CList: ERROR! Invalid range for firstMatch"; + + fprintf(stderr, "%s - shift '%zu', size '%zu', item_size '%zu'\n", errmsg, shift, size, p->item_size); + assert(shift + size <= p->item_size); + return NULL; + } + + if (shift == 0 && size == 0) + size = p->item_size; + + size_t i = shift; + size_t end = p->count * step; + int index = 0; + + if (string) { + for (; i < end; i += step, index++) { + if (strncmp(data + i, o, size) == 0) { + p->lastSearchPos = index; + return (data + i - shift); + } + } + } else { + for (; i < end; i += step, index++) { + if (memcmp(data + i, o, size) == 0) { + p->lastSearchPos = index; + return (data + i - shift); + } + } + } + + return NULL; +} + +void *CList_lastMatch_(struct CList *l, const void *o, size_t shift, size_t size, int string) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + char *data = (char *) p->items; + size_t step = p->item_size; + + p->lastSearchPos = -1; + + if (shift + size > p->item_size) { + fprintf(stderr, "CList: ERROR! Wrong ranges for lastMatch - shift '%zu', size '%zu', item_size '%zu'\n", + shift, size, p->item_size); + assert(shift + size <= p->item_size); + return NULL; + } + + if (shift == 0 && size == 0) + size = p->item_size; + + int index = p->count - 1; + long i = index * step + shift; + + if (string) { + for (; i >= 0; i -= step, index--) { + if (strncmp(data + i, o, size) == 0) { + p->lastSearchPos = index; + return (data + i - shift); + } + } + } else { + for (; i >= 0; i -= step, index--) { + if (memcmp(data + i, o, size) == 0) { + p->lastSearchPos = index; + return (data + i - shift); + } + } + } + + return NULL; +} + +int CList_index_(CList *l) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + return p->lastSearchPos; +} + +int CList_swap_(CList *l, int a, int b) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + if (a < 0 || a >= p->count || b < 0 || b >= p->count) { + fprintf(stderr, "CList: ERROR! Swap position outside range - %i, %i; count - %d.\n", + a, b, p->count); + assert(a >= 0 && a < p->count && b >= 0 && b < p->count); + return 0; + } + + if (a == b) + return 1; /* ? Good ? :D */ + + char *data = (char *) p->items; + size_t step = p->item_size; + + if (p->count == p->alloc_size && + CList_Realloc_(l, p->alloc_size + 1) == 0) + return 0; + + memcpy(data + p->count * step, data + a * step, step); + memcpy(data + a * step, data + b * step, step); + memcpy(data + b * step, data + p->count * step, step); + return 1; +} + +int CList_Count_(CList *l) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + return p->count; +} + +int CList_AllocSize_(CList *l) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + return p->alloc_size; +} + +size_t CList_ItemSize_(CList *l) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + return p->item_size; +} + +void CList_Clear_(CList *l) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + free(p->items); + p->items = NULL; + p->alloc_size = 0; + p->count = 0; +} + +void CList_Free_(CList *l) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + free(p->items); + free(p); + free(l); + l = NULL; +} + +void CList_print_(CList *l, size_t shift, int n, const char *type) +{ + CList_priv_ *p = (CList_priv_ *) l->priv; + + if (shift >= p->item_size) { + fprintf(stderr, "CList: ERROR! Wrong shift value for list print - shift '%zu', item_size '%zu'\n", + shift, p->item_size); + assert(shift < p->item_size); + return; + } + + printf("\nCList: count = %i item_size = %zu alloc_size = %i type = %s\n", + p->count, p->item_size, p->alloc_size, type); + + if (n > 0) { + int tp = -1; + + if (type == NULL) + tp = 0; /* Print out pointers */ + else if (strcmp(type, "char") == 0) + tp = 1; + else if (strcmp(type, "short") == 0) + tp = 2; + else if (strcmp(type, "int") == 0) + tp = 3; + else if (strcmp(type, "long") == 0) + tp = 4; + else if (strcmp(type, "uintptr_t") == 0) + tp = 5; + else if (strcmp(type, "size_t") == 0) + tp = 6; + else if (strcmp(type, "double") == 0) + tp = 7; + else if (strcmp(type, "string") == 0) + tp = 8; + + if (tp == -1) { + fprintf(stderr, "CList: Can not print - not supported type - %s\n\n", type); + return; + } + + n = (n > p->count) ? p->count : n; + char *data = (char *) p->items + shift; + size_t step = p->item_size; + int i = 0; + + for (; i < n; i++) { + switch (tp) { + case 0: + printf("%p ", data); break; + case 1: + printf("%c ", *(char *) data); break; + case 2: + printf("%hi ", *(short *) data); break; + case 3: + printf("%i ", *(int *) data); break; + case 4: + printf("%li ", *(long *) data); break; + case 5: + printf("0x%lx ", *(uintptr_t *) data); break; + case 6: + printf("%zu ", *(size_t *) data); break; + case 7: + printf("%f ", *(double *) data); break; + case 8: + printf("%s\n", data); break; + default: + return; + } + + data += step; + } + printf("\n\n"); + } +} + +CList *CList_init(size_t objSize) +{ + CList *lst = malloc(sizeof(CList)); + CList_priv_ *p = malloc(sizeof(CList_priv_)); + + if (!lst || !p) { + fprintf(stderr, "CList: ERROR! Can not allocate CList!\n"); + return NULL; + } + p->count = 0; + p->alloc_size = 0; + p->lastSearchPos = -1; + p->item_size = objSize; + p->items = NULL; + lst->add = &CList_Add_; + lst->insert = &CList_Insert_; + lst->replace = &CList_Replace_; + lst->remove = &CList_Remove_; + lst->at = &CList_At_; + lst->realloc = &CList_Realloc_; + lst->count = &CList_Count_; + lst->firstMatch = &CList_firstMatch_; + lst->lastMatch = &CList_lastMatch_; + lst->index = &CList_index_; + lst->swap = &CList_swap_; + lst->allocSize = &CList_AllocSize_; + lst->itemSize = &CList_ItemSize_; + lst->print = &CList_print_; + lst->clear = &CList_Clear_; + lst->free = &CList_Free_; + lst->priv = p; + return lst; +} + diff --git a/content/chapters/compute/lab/support/CLIST/clist.h b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/clist.h similarity index 100% rename from content/chapters/compute/lab/support/CLIST/clist.h rename to chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/clist.h diff --git a/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/test.c b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/test.c new file mode 100644 index 0000000000..04af11c909 --- /dev/null +++ b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/test.c @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: BSD-2-Clause + +/******************************************/ +/* */ +/* Alexander Agdgomlishvili */ +/* */ +/* cdevelopment@mail.com */ +/* */ +/******************************************/ + +/** + * BSD 2-Clause License + * + * Copyright (c) 2020, Alexander Agdgomlishvili + * cdevelopment@mail.com + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* Github link: https://github.com/AlexanderAgd/CLIST */ + +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) +#include +#endif + +#if defined(_POSIX_VERSION) +#include +#elif defined(WIN32) +#include +#endif + +#include +#include +#include +#include +#include +#include "clist.h" + +#if defined(WIN32) +int gettimeofday(struct timeval *tp, void *tzp); +#endif + +size_t diff_usec(struct timeval start, struct timeval end) +{ + return (1000000 * (end.tv_sec - start.tv_sec) + end.tv_usec - start.tv_usec); +} + +int main(void) +{ + void *obj = NULL; + int i = 0; + int n = 0; + + /******************************************************* CHAR LIST */ + + n = sizeof(char); + CList *lst = CList_init(n); + + for (i = 33; i < 123; i++) { + obj = &i; + lst->add(lst, obj); /* Adding obj items to the end of array */ + } + lst->print(lst, 0, 150, "char"); + + n = 32; + char ch = 'A'; + + obj = lst->at(lst, n); /* Get 32nd item of array */ + printf("Position %i contains symbol \'%c\'\n", n, *((char *)obj)); + + ch = '!'; + obj = &ch; + lst->firstMatch(lst, obj, 0, 0, 0); /* Find first match of '!' char */ + /* You can skip (set to zero) "shift" and "size" parameters when compare whole item */ + n = lst->index(lst); + printf("Index of \'%c\' is %i\n", ch, n); + + ch = '+'; + lst->firstMatch(lst, obj, 0, 0, 0); + n = lst->index(lst); + printf("Index of \'%c\' is %i\n", ch, n); + + n = 0; + for (i = 15; i > 0; i--) + lst->remove(lst, 0); /* Remove item at index 0 from array 15 times */ + + lst->print(lst, 0, 150, "char"); + lst->free(lst); + + /******************************************************* SHORT INT LIST */ + + n = sizeof(short); + lst = CList_init(n); /* Always set object size you will work with */ + + short sh = 1001; + + for (i = 0; i < 24; i++, sh += 1000) { + obj = &sh; + lst->add(lst, obj); + } + + lst->print(lst, 0, 100, "short"); + + sh = 5001; + lst->insert(lst, &sh, 20); /* Insert value of 'sh' to position 20 */ + + lst->insert(lst, &sh, 25); /* Insert value of 'sh' to position 25 */ + lst->print(lst, 0, lst->count(lst), "short"); + + lst->lastMatch(lst, &sh, 0, 0, 0); /* Find last match of '5001' short */ + n = lst->index(lst); + printf("Last index of \'%i\' is %i\n", sh, n); + + while (n != -1) { + lst->firstMatch(lst, &sh, 0, 0, 0); /* Find first match of '5001' short */ + n = lst->index(lst); + + if (n != -1) /* Remove it from the list */ + lst->remove(lst, n); + } + + sh = 1111; + lst->replace(lst, &sh, 0); /* Replace object at position '0' */ + lst->print(lst, 0, 100, "short"); + + lst->clear(lst); /* CLear list */ + lst->free(lst); + + /******************************************************* STRUCT LIST */ + + struct sample { + long l; + double d; + int s; + char info[24]; + union u { + long long lli; + char c; + } u; + union u *dat; + } sample; + + n = sizeof(sample); + lst = CList_init(n); + + printf("Size of struct 'sample' = %zu bytes\n", sizeof(sample)); + + n = 0; + for (i = 0; i < 8; i++) { /* Let create list of a struct */ + struct sample sam = { + .l = 10 * i, + .d = 5.010101 * i, + .s = i, + .info = "", + .u = { .lli = 11 * i }, + .dat = &(sam.u) + }; + + sprintf(sam.info, "Some string_%c", 80 + i); + + obj = &sam; + lst->insert(lst, obj, n); /* Insert each object at index '0' */ + } + + lst->print(lst, 0, 20, NULL); + + int j; + int count = lst->count(lst); + + for (j = 0; j < 6; j++, i = 0) { /* Print out struct content */ + for (i = 0; i < count; i++) { + struct sample *sam = lst->at(lst, i); + + switch (j) { + case 0: + printf("%15li ", sam->l); break; + case 1: + printf("%15lf ", sam->d); break; + case 2: + printf("%15i ", sam->s); break; + case 3: + printf("%15s ", sam->info); break; + case 4: + printf("%15lli ", sam->u.lli); break; + case 5: + printf("%15p ", sam->dat); break; + default: + break; + } + } + printf("\n\n"); + } + + /* Advanced usage of firstMatch and lastMatch */ + + size_t shift = offsetof(struct sample, info); + size_t size = sizeof(sample.info); + int its_a_string = 1; + struct sample *smpl = lst->firstMatch(lst, "Some string_R", shift, size, its_a_string); + + n = lst->index(lst); + + printf("List item with index %i contains string \"%s\"\n", n, smpl->info); + + long long longValue = 55; + + shift = offsetof(struct sample, u); + size = sizeof(long long); + int string = 0; + + smpl = lst->firstMatch(lst, &longValue, shift, size, string); + n = lst->index(lst); + + printf("List item with index %i contains long long int \"%lli\"\n", n, smpl->u.lli); + + /* Print struct member */ + + shift = offsetof(struct sample, info); + lst->print(lst, shift, 8, "string"); + lst->free(lst); + + /******************************************************* POINTERS LIST */ + + /* If we want create list of objects located in diffent places of memory, */ + /* let create pointers list or in our case "uintptr_t" address list */ + + printf("Size of 'uintptr_t' = %zu bytes\n", sizeof(uintptr_t)); + + n = sizeof(uintptr_t); + lst = CList_init(n); + + struct sample sm1 = { 64, 6.4, 4, "ABC company", { 16 }, obj }; + struct sample sm2 = { 128, 12.8, 8, "Discovery", { 1024 }, (void *)&sh }; /* Just some sample data */ + + uintptr_t addr = (uintptr_t) &sm1; /* Cast reference to address value */ + + lst->add(lst, &addr); + + addr = (uintptr_t) &sm2; + lst->add(lst, &addr); + + struct sample *sm3 = malloc(sizeof(sample)); + + sm3->l = 256; + sm3->d = 25.6; + sm3->s = 16; + strcpy(sm3->info, "TV show"); + sm3->u.lli = 2048; + sm3->dat = (void *) &sm2; + + addr = (uintptr_t) sm3; + lst->add(lst, &addr); + + lst->print(lst, 0, 20, "uintptr_t"); + + count = lst->count(lst); + for (j = 0; j < 6; j++, i = 0) { /* Print out struct content */ + for (i = 0; i < count; i++) { + uintptr_t *ad = lst->at(lst, i); + struct sample *sam = (struct sample *) *ad; /* Cast address value to struct pointer */ + + switch (j) { + case 0: + printf("%15li ", sam->l); break; + case 1: + printf("%15lf ", sam->d); break; + case 2: + printf("%15i ", sam->s); break; + case 3: + printf("%15s ", sam->info); break; + case 4: + printf("%15lli ", sam->u.lli); break; + case 5: + printf("%15p ", sam->dat); break; + default: + break; + } + } + printf("\n"); + } + + printf("\n"); + //lst->print(lst, shift, 3, "uintptr_t"); + free(sm3); + lst->free(lst); + + /******************************************************* PERFORMANCE TEST */ + + n = sizeof(int); + lst = CList_init(n); + + size_t time; + + i = 0; + n = 10000; + struct timeval start; + struct timeval end; + + printf("PERFORMANCE TEST - 1 second contains 1000000 microseconds\n\n"); + + gettimeofday(&start, NULL); + for (i = 0; i < n; i++) + lst->add(lst, &i); + gettimeofday(&end, NULL); + time = diff_usec(start, end); + printf("Add of %i int takes - %zu microseconds\n", n, time); + + gettimeofday(&start, NULL); + for (i = 0; i < n; i++) + lst->remove(lst, 0); + gettimeofday(&end, NULL); + time = diff_usec(start, end); + printf("Remove from position '0' of %i int takes - %zu microseconds\n", n, time); + + gettimeofday(&start, NULL); + for (i = 0; i < n; i++) + lst->insert(lst, &i, 0); + gettimeofday(&end, NULL); + time = diff_usec(start, end); + printf("Insert to position '0' of %i int takes - %zu microseconds\n", n, time); + + gettimeofday(&start, NULL); + for (i = 0; i < n; i++) + lst->remove(lst, lst->count(lst) - 1); + gettimeofday(&end, NULL); + time = diff_usec(start, end); + printf("Remove from last position of %i int takes - %zu microseconds\n", n, time); + + gettimeofday(&start, NULL); + for (i = 0; i < n; i++) + lst->insert(lst, &i, lst->count(lst)); + gettimeofday(&end, NULL); + time = diff_usec(start, end); + printf("Insert to last position of %i int takes - %zu microseconds\n", n, time); + + gettimeofday(&start, NULL); + for (i = 0; i < n; i++) + lst->replace(lst, &n, i); + gettimeofday(&end, NULL); + time = diff_usec(start, end); + printf("Replace of %i int takes - %zu microseconds\n", n, time); + + printf("\n"); + lst->free(lst); + + return 0; + +} /**** MAIN ****/ + +#if defined(WIN32) +int gettimeofday(struct timeval *tp, void *tzp) +{ + LARGE_INTEGER ticksPerSecond; + LARGE_INTEGER tick; + LARGE_INTEGER time; + + if (QueryPerformanceFrequency(&ticksPerSecond) == 0) + return -1; + QueryPerformanceCounter(&tick); /* what time is it? */ + + time.QuadPart = tick.QuadPart / ticksPerSecond.QuadPart; /* convert the tick number into the number */ + /* of seconds since the system was started... */ + tp->tv_sec = time.QuadPart; /* seconds */ + tp->tv_usec = (tick.QuadPart - (time.QuadPart * ticksPerSecond.QuadPart)) * + 1000000.0 / ticksPerSecond.QuadPart; + return 0; +} +#endif diff --git a/content/chapters/compute/lab/support/CLIST/test_parallel.c b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/test_parallel.c similarity index 98% rename from content/chapters/compute/lab/support/CLIST/test_parallel.c rename to chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/test_parallel.c index 8d23a29ebb..2c97c4bf3d 100644 --- a/content/chapters/compute/lab/support/CLIST/test_parallel.c +++ b/chapters/compute/synchronization/drills/tasks/threadsafe-data-struct/support/test_parallel.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: BSD-3-Clause + #include #include #include @@ -14,7 +16,7 @@ #define INT_SIZE sizeof(int) static size_t diff_usec(struct timeval start, struct timeval end) -{ +{ return (1000000 * (end.tv_sec - start.tv_sec) + end.tv_usec - start.tv_usec); } @@ -82,12 +84,12 @@ static void *replace_same_from_list(void *arg) static int init_even_elem(size_t idx) { - return idx * 2; + return idx * 2; } static int init_odd_elem(size_t idx) { - return idx * 2 + 1; + return idx * 2 + 1; } static int init_remove_elem(size_t idx) diff --git a/chapters/compute/synchronization/guides/apache2-simulator/README.md b/chapters/compute/synchronization/guides/apache2-simulator/README.md new file mode 100644 index 0000000000..a052a18fc0 --- /dev/null +++ b/chapters/compute/synchronization/guides/apache2-simulator/README.md @@ -0,0 +1,74 @@ +# `apache2` Simulator + +## Semaphore + +Go to `apache2-simulator/support/apache2_simulator_semaphore.py`. +In the `main()` function we create a semaphore which we increment (`release()`) upon every new message. +Each thread decrements (`acquire()`) this semaphore to signal that it wants to retrieve a message from the list. +The retrieval means modifying a data structure, which is a critical section, so we use a **separate** mutex for this. +Otherwise, multiple threads could acquire the semaphore at the same time and try to modify the list at the same time. +Not good. + +Locking this mutex (which in Python is called `Lock`) is done with the following statement: `with msg_mutex:` +This is a syntactic equivalent to: + +```Python +event.acquire() +messages.append(msg) +event.release() +``` + +[Quiz](../../drills/questions/semaphore-equivalent.md) + +Since the length of the `messages` list is simply `len(messages)`, it may seem a bit redundant to use a semaphore to store essentially the same value. +In the next section, we'll look at a more refined mechanism for our use case: _condition variables_. + +## Condition + +But this is not all, unfortunately. +Look at the code in `apache2-simulator/support/apache2_simulator_condition.py`. +See the main thread call `notify()` once it reads the message. +Notice that this call is preceded by an `acquire()` call, and succeeded by a `release()` call. + +`acquire()` and `release()` are commonly associated with mutexes or semaphores. +What do they have to do with condition variables? + +Well, a lock `Condition` variable also stores an inner lock (mutex). +It is this lock that we `acquire()` and `release()`. +In fact, the [documentation](https://docs.python.org/3/library/threading.html#condition-objects) states we should only call `Condition` methods with its inner lock taken. + +Why is this necessary? +Take a look at the `worker()` function. +After `wait()`-ing (we'll explain the need for the loop in a bit), it extracts a message from the message queue. +This operation is **not** atomic, so it must be enclosed within a critical section. +Hence, the lock. + +[Quiz](../../drills/questions/notify-only-with-mutex.md) + +So now we know we cannot only use a mutex. +The mutex is used to access and modify the `messages` list atomically. +Now, you might be thinking that this code causes a deadlock: + +```Python +event.acquire() +while len(messages) == 0: + event.wait() +``` + +The thread gets the lock and then, if there are no messages, it switches its state to WAITING. +A classic deadlock, right? +No. +`wait()` also releases the inner lock of the `Condition` and being woken up reacquires it. +Neat! +And the `while` loop that checks if there are any new messages is necessary because `wait()` can return after an arbitrary long time. +This is because the thread waiting for the event was notified to wake up, but another thread has woken up before it and started handling the event earlier by reacquiring the lock. +All the other threads that woke up, but can't acquire the lock, must be put back to wait. +This situation is called a **spurious wakeup**. +Therefore, it's necessary to check for messages again when waking up. + +So now we have both synchronization **and** signalling. +This is what conditions are for, ultimately. + +Now that you understand the concept of synchronization, you should apply it in a broader context. +[In the Arena], you'll find an exercise asking you to make an existing array list implementation thread-safe. +Have fun! diff --git a/content/chapters/compute/lab/support/apache2-simulator/apache2_simulator_condition.py b/chapters/compute/synchronization/guides/apache2-simulator/support/apache2_simulator_condition.py similarity index 100% rename from content/chapters/compute/lab/support/apache2-simulator/apache2_simulator_condition.py rename to chapters/compute/synchronization/guides/apache2-simulator/support/apache2_simulator_condition.py diff --git a/content/chapters/compute/lab/support/apache2-simulator/apache2_simulator_semaphore.py b/chapters/compute/synchronization/guides/apache2-simulator/support/apache2_simulator_semaphore.py similarity index 100% rename from content/chapters/compute/lab/support/apache2-simulator/apache2_simulator_semaphore.py rename to chapters/compute/synchronization/guides/apache2-simulator/support/apache2_simulator_semaphore.py diff --git a/content/chapters/compute/lecture/media/deadlock.svg b/chapters/compute/synchronization/media/deadlock.svg similarity index 100% rename from content/chapters/compute/lecture/media/deadlock.svg rename to chapters/compute/synchronization/media/deadlock.svg diff --git a/content/chapters/compute/lecture/media/lock-undefined-behaviour.svg b/chapters/compute/synchronization/media/lock-undefined-behaviour.svg similarity index 100% rename from content/chapters/compute/lecture/media/lock-undefined-behaviour.svg rename to chapters/compute/synchronization/media/lock-undefined-behaviour.svg diff --git a/content/chapters/compute/lecture/media/lock-vs-notify.svg b/chapters/compute/synchronization/media/lock-vs-notify.svg similarity index 100% rename from content/chapters/compute/lecture/media/lock-vs-notify.svg rename to chapters/compute/synchronization/media/lock-vs-notify.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-0.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-0.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-0.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-0.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-1.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-1.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-1.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-1.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-10.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-10.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-10.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-10.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-2.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-2.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-2.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-2.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-3.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-3.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-3.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-3.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-4.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-4.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-4.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-4.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-5.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-5.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-5.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-5.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-6.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-6.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-6.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-6.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-7.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-7.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-7.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-7.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-8.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-8.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-8.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-8.svg diff --git a/content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-9.svg b/chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-9.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-lock/race-condition-lock-9.svg rename to chapters/compute/synchronization/media/race-condition-lock/race-condition-lock-9.svg diff --git a/content/chapters/compute/lecture/media/race-condition-toctou/race-condition-toctou-0.svg b/chapters/compute/synchronization/media/race-condition-toctou/race-condition-toctou-0.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-toctou/race-condition-toctou-0.svg rename to chapters/compute/synchronization/media/race-condition-toctou/race-condition-toctou-0.svg diff --git a/content/chapters/compute/lecture/media/race-condition-toctou/race-condition-toctou-1.svg b/chapters/compute/synchronization/media/race-condition-toctou/race-condition-toctou-1.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-toctou/race-condition-toctou-1.svg rename to chapters/compute/synchronization/media/race-condition-toctou/race-condition-toctou-1.svg diff --git a/content/chapters/compute/lecture/media/race-condition-toctou/race-condition-toctou-2.svg b/chapters/compute/synchronization/media/race-condition-toctou/race-condition-toctou-2.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-toctou/race-condition-toctou-2.svg rename to chapters/compute/synchronization/media/race-condition-toctou/race-condition-toctou-2.svg diff --git a/content/chapters/compute/lecture/media/race-condition-toctou/race-condition-toctou-3.svg b/chapters/compute/synchronization/media/race-condition-toctou/race-condition-toctou-3.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-toctou/race-condition-toctou-3.svg rename to chapters/compute/synchronization/media/race-condition-toctou/race-condition-toctou-3.svg diff --git a/content/chapters/compute/lecture/media/race-condition-toctou/race-condition-toctou-4.svg b/chapters/compute/synchronization/media/race-condition-toctou/race-condition-toctou-4.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition-toctou/race-condition-toctou-4.svg rename to chapters/compute/synchronization/media/race-condition-toctou/race-condition-toctou-4.svg diff --git a/content/chapters/compute/lecture/media/race-condition/race-condition-0.svg b/chapters/compute/synchronization/media/race-condition/race-condition-0.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition/race-condition-0.svg rename to chapters/compute/synchronization/media/race-condition/race-condition-0.svg diff --git a/content/chapters/compute/lecture/media/race-condition/race-condition-1.svg b/chapters/compute/synchronization/media/race-condition/race-condition-1.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition/race-condition-1.svg rename to chapters/compute/synchronization/media/race-condition/race-condition-1.svg diff --git a/content/chapters/compute/lecture/media/race-condition/race-condition-2.svg b/chapters/compute/synchronization/media/race-condition/race-condition-2.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition/race-condition-2.svg rename to chapters/compute/synchronization/media/race-condition/race-condition-2.svg diff --git a/content/chapters/compute/lecture/media/race-condition/race-condition-3.svg b/chapters/compute/synchronization/media/race-condition/race-condition-3.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition/race-condition-3.svg rename to chapters/compute/synchronization/media/race-condition/race-condition-3.svg diff --git a/content/chapters/compute/lecture/media/race-condition/race-condition-4.svg b/chapters/compute/synchronization/media/race-condition/race-condition-4.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition/race-condition-4.svg rename to chapters/compute/synchronization/media/race-condition/race-condition-4.svg diff --git a/content/chapters/compute/lecture/media/race-condition/race-condition-5.svg b/chapters/compute/synchronization/media/race-condition/race-condition-5.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition/race-condition-5.svg rename to chapters/compute/synchronization/media/race-condition/race-condition-5.svg diff --git a/content/chapters/compute/lecture/media/race-condition/race-condition-6.svg b/chapters/compute/synchronization/media/race-condition/race-condition-6.svg similarity index 100% rename from content/chapters/compute/lecture/media/race-condition/race-condition-6.svg rename to chapters/compute/synchronization/media/race-condition/race-condition-6.svg diff --git a/content/chapters/compute/lecture/media/recursive-vs-reentrant.svg b/chapters/compute/synchronization/media/recursive-vs-reentrant.svg similarity index 100% rename from content/chapters/compute/lecture/media/recursive-vs-reentrant.svg rename to chapters/compute/synchronization/media/recursive-vs-reentrant.svg diff --git a/content/chapters/compute/lab/content/synchronization.md b/chapters/compute/synchronization/reading/synchronization.md similarity index 66% rename from content/chapters/compute/lab/content/synchronization.md rename to chapters/compute/synchronization/reading/synchronization.md index a423e58b7b..f351fe6ce2 100644 --- a/content/chapters/compute/lab/content/synchronization.md +++ b/chapters/compute/synchronization/reading/synchronization.md @@ -31,7 +31,7 @@ But this is up to the scheduler and is non-deterministic. Such undefined behaviours can cripple the execution of a program if `var` is some critical variable. Let's see this bug in action. -Go to `support/race-condition/c/race_condition.c`, compile and run the code a few times. +Go to `race-condition/support/c/race_condition.c`, compile and run the code a few times. It spawns to threads that do exactly what we've talked about so far: one thread increments `var` 10 million times, while the other decrements it 10 million times. As you can see from running the program, the differences between subsequent runs can be substantial. @@ -41,7 +41,7 @@ A critical section is a piece of code that can only be executed by **one thread* So we need some sort of _mutual exclusion mechanism_ so that when one thread runs the critical section, the other has to **wait** before entering it. This mechanism is called a **mutex**, whose name comes from "mutual exclusion". -Go to `support/race-condition/c/race_condition_mutex.c` and notice the differences between this code and the buggy one. +Go to `race-condition/support/c/race_condition_mutex.c` and notice the differences between this code and the buggy one. We now use a `pthread_mutex_t` variable, which we `lock` at the beginning of a critical section, and we `unlock` at the end. Generally speaking, `lock`-ing a mutex makes a thread enter a critical section, while calling `pthread_mutex_unlock()` makes the thread leave said critical section. Therefore, as we said previously, the critical sections in our code are `var--` and `var++`. @@ -81,7 +81,7 @@ This latter overhead comes from the **context switch** that is necessary for a t Move the calls to `pthread_mutex_lock()` and `pthread_mutex_unlock()` outside the `for` statements so that the critical sections become the entire statement. Measure the new time spent by the code and compare it with the execution times recorded when the critical sections were made up of only `var--` and `var++`. -[Quiz](../quiz/coarse-vs-granular-critical-section.md) +[Quiz](../drills/questions/coarse-vs-granular-critical-section.md) ## Atomics @@ -98,7 +98,7 @@ Modern processors are capable of _atomically_ accessing data, either for reads o An atomic action is an indivisible sequence of operations that a thread runs without interference from others. Concretely, before initiating an atomic transfer on one of its data buses, the CPU first makes sure all other transfers have ended, then **locks** the data bus by stalling all cores attempting to transfer data on it. This way, one thread obtains **exclusive** access to the data bus while accessing data. -As a side note, the critical sections in `support/race-condition/c/race_condition_mutex.c` are also atomic once they are wrapped between calls to `pthread_mutex_lock()` and `pthread_mutex_unlock()`. +As a side note, the critical sections in `race-condition/support/c/race_condition_mutex.c` are also atomic once they are wrapped between calls to `pthread_mutex_lock()` and `pthread_mutex_unlock()`. As with every hardware feature, the `x86` ISA exposes an instruction for atomic operations. In particular, this instruction is a **prefix**, called `lock`. @@ -107,20 +107,20 @@ The `lock` prefix ensures that the core performing the instruction has exclusive This is how the increment is made into an indivisible unit. For example, `inc dword [x]` can be made atomic, like so: `lock inc dword [x]`. -You can play with the `lock` prefix [in the Arena](./arena.md#atomic-assembly). +You can play with the `lock` prefix [in the Arena](atomic-assembly). Compilers provide support for such hardware-level atomic operations. GCC exposes [built-ins](https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html) such as `__atomic_load()`, `__atomic_store()`, `__atomic_compare_exchange()` and many others. All of them rely on the mechanism described above. -Go to `support/race-condition/c/race_condition_atomic.c` and complete the function `decrement_var()`. +Go to `race-condition/support/c/race_condition_atomic.c` and complete the function `decrement_var()`. Compile and run the code. Now measure its running time against the mutex implementations. It should be somewhere between `race_condition.c` and `race_condition_mutex.c`. The C standard library also provides atomic data types. Access to these variables can be done only by one thread at a time. -Go to `support/race-condition/c/race_condition_atomic2.c`, compile and run the code. +Go to `race-condition/support/c/race_condition_atomic2.c`, compile and run the code. Now measure its running time against the other implementations. Notice that the time is similar to `race_condition_atomic`. @@ -132,12 +132,12 @@ And the fact that high-level languages also expose an API for atomic operations Up to now, we've learned how to create critical sections that can be accessed by **only one thread** at a time. These critical sections revolved around **data**. Whenever we define a critical section, there is some specific data to which we cannot allow parallel access. -The reason why we can't allow it is, in general, data integrity, as we've seen in our examples in `support/race-condition/` +The reason why we can't allow it is, in general, data integrity, as we've seen in our examples in `race-condition/support/` But what if threads need to count? Counting is inherently thread-unsafe because it's a _read-modify-write_ operation. We read the counter, increment (modify) it and then write it back. -Think about our example with [`apache2`](./processes-threads-apache2.md) +Think about our example with [`apache2`](../../process-threads-apache2/reading/processes-threads-apache2.md) Let's say a `worker` has created a _pool_ of 3 threads. They are not doing any work initially; they are in the WAITING state. @@ -153,29 +153,6 @@ When a thread attempts to `acquire()` a semaphore, it will wait if this counter Otherwise, the thread **decrements** the internal counter and the function returns. The opposite of `acquire()` is `release()`, which increases the internal counter by a given value (by default 1). -### Practice: `apache2` Simulator - Semaphore - -Go to `support/apache2-simulator/apache2_simulator_semaphore.py`. -In the `main()` function we create a semaphore which we increment (`release()`) upon every new message. -Each thread decrements (`acquire()`) this semaphore to signal that it wants to retrieve a message from the list. -The retrieval means modifying a data structure, which is a critical section, so we use a **separate** mutex for this. -Otherwise, multiple threads could acquire the semaphore at the same time and try to modify the list at the same time. -Not good. - -Locking this mutex (which in Python is called `Lock`) is done with the following statement: `with msg_mutex:` -This is a syntactic equivalent to: - -```Python -event.acquire() -messages.append(msg) -event.release() -``` - -[Quiz](../quiz/semaphore-equivalent.md) - -Since the length of the `messages` list is simply `len(messages)`, it may seem a bit redundant to use a semaphore to store essentially the same value. -In the next section, we'll look at a more refined mechanism for our use case: _condition variables_. - ## Conditions Another way we can implement our `apache2` simulator is to use a condition variable. @@ -189,56 +166,6 @@ As you might expect, they are complementary: - `notify()` wakes up one or more `wait()`-ing threads. If `notify()` is called before any thread has called `wait()`, the first thread that calls it will continue its execution unhindered. -### Practice: `apache2` Simulator - Condition - -But this is not all, unfortunately. -Look at the code in `support/apache2-simulator/apache2_simulator_condition.py`. -See the main thread call `notify()` once it reads the message. -Notice that this call is preceded by an `acquire()` call, and succeeded by a `release()` call. - -`acquire()` and `release()` are commonly associated with mutexes or semaphores. -What do they have to do with condition variables? - -Well, a lock `Condition` variable also stores an inner lock (mutex). -It is this lock that we `acquire()` and `release()`. -In fact, the [documentation](https://docs.python.org/3/library/threading.html#condition-objects) states we should only call `Condition` methods with its inner lock taken. - -Why is this necessary? -Take a look at the `worker()` function. -After `wait()`-ing (we'll explain the need for the loop in a bit), it extracts a message from the message queue. -This operation is **not** atomic, so it must be enclosed within a critical section. -Hence, the lock. - -[Quiz](../quiz/notify-only-with-mutex.md) - -So now we know we cannot only use a mutex. -The mutex is used to access and modify the `messages` list atomically. -Now, you might be thinking that this code causes a deadlock: - -```Python -event.acquire() -while len(messages) == 0: - event.wait() -``` - -The thread gets the lock and then, if there are no messages, it switches its state to WAITING. -A classic deadlock, right? -No. -`wait()` also releases the inner lock of the `Condition` and being woken up reacquires it. -Neat! -And the `while` loop that checks if there are any new messages is necessary because `wait()` can return after an arbitrary long time. -This is because the thread waiting for the event was notified to wake up, but another thread has woken up before it and started handling the event earlier by reacquiring the lock. -All the other threads that woke up, but can't acquire the lock, must be put back to wait. -This situation is called a **spurious wakeup**. -Therefore, it's necessary to check for messages again when waking up. - -So now we have both synchronization **and** signalling. -This is what conditions are for, ultimately. - -Now that you understand the concept of synchronization, you should apply it in a broader context. -[In the Arena](./arena.md#synchronization---thread-safe-data-structure), you'll find an exercise asking you to make an existing array list implementation thread-safe. -Have fun! - ## Thread-Local Storage (TLS) First things first: what if we don't want data to be shared between threads? @@ -249,27 +176,3 @@ To protect data from race conditions "by design", we can place in what's called As its name implies, this is a type of storage that is "owned" by individual threads, as opposed to being shared among all threads. **Do not confuse it with copy-on-write**. TLS pages are always duplicated when creating a new thread and their contents are reinitialised. - -### Practice: C - TLS on Demand - -The perspective of C towards TLS is the following: everything is shared by default. -This makes multithreading easier and more lightweight to implement than in other languages, like D, because synchronization is left entirely up to the developer, at the cost of potential unsafety. - -Of course, we can specify that some data belongs to the TLS, by preceding the declaration of a variable with `__thread` keyword. -First, compile and run the code in `support/race-condition/c/race_condition_tls.c` a few times. -As expected, the result is different each time. - -1. Modify the declaration of `var` and add the `__thread` keyword to place the variable in the TLS of each thread. -Recompile and run the code a few more times. -You should see that in the end, `var` is 0. - -[Quiz 1](../quiz/tls-synchronization.md) - -[Quiz 2](../quiz/tls-var-copies.md) - -1. Print the address and value of `var` in each thread. -See that they differ. - -1. Modify the value of `var` in the `main()` function before calling `pthread_create()`. -Notice that the value doesn't propagate to the other threads. -This is because, upon creating a new thread, its TLS is initialised. diff --git a/content/chapters/compute/lecture/slides/barrier.md b/chapters/compute/synchronization/slides/barrier.md similarity index 100% rename from content/chapters/compute/lecture/slides/barrier.md rename to chapters/compute/synchronization/slides/barrier.md diff --git a/content/chapters/compute/lecture/slides/mutual-exclusion.md b/chapters/compute/synchronization/slides/mutual-exclusion.md similarity index 94% rename from content/chapters/compute/lecture/slides/mutual-exclusion.md rename to chapters/compute/synchronization/slides/mutual-exclusion.md index 31db1b74bd..d6d0b737d9 100644 --- a/content/chapters/compute/lecture/slides/mutual-exclusion.md +++ b/chapters/compute/synchronization/slides/mutual-exclusion.md @@ -8,11 +8,11 @@ * Access to a resource is restricted to a single thread * Use **atomic** instructions provided by the ISA: (on x86: the `lock` prefix) - * Eg: `lock inc dword [var]` + * e.g.: `lock inc dword [var]` * The CPU core has exclusive access to the cache and data bus while executing the instruction * Use architecture-independent wrappers: - * [GCC built-ins](https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html) (eg. `__atomic_add_fetch`) - * libc functions from [`stdatomic.h`](https://en.cppreference.com/w/c/atomic) (eg. `atomic_flag_test_and_set`) + * [GCC built-ins](https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html) (e.g.. `__atomic_add_fetch`) + * libc functions from [`stdatomic.h`](https://en.cppreference.com/w/c/atomic) (e.g.. `atomic_flag_test_and_set`) ---- @@ -103,7 +103,7 @@ void mutex_unlock(struct mutex *m) #### Who Should Release the Lock? -![Locks Undefined Behaviour](./media/lock-undefined-behaviour.svg) +![Locks Undefined Behaviour](../media/lock-undefined-behaviour.svg) --- @@ -158,7 +158,7 @@ var = 2000000; time = 1268 ms * Circular dependency * Threads cyclically wait for each other -![Deadlock](./media/deadlock.svg) +![Deadlock](../media/deadlock.svg) ```console student@os:~/.../compute/lecture/demo/deadlock$ python3 deadlock.py diff --git a/content/chapters/compute/lecture/slides/notifications.md b/chapters/compute/synchronization/slides/notifications.md similarity index 95% rename from content/chapters/compute/lecture/slides/notifications.md rename to chapters/compute/synchronization/slides/notifications.md index 0ef1b85eaa..0b5a9e19e4 100644 --- a/content/chapters/compute/lecture/slides/notifications.md +++ b/chapters/compute/synchronization/slides/notifications.md @@ -6,7 +6,7 @@ ### Locks vs Notifications -![Lock vs Notification](./media/lock-vs-notify.svg) +![Lock vs Notification](../media/lock-vs-notify.svg) ---- @@ -32,7 +32,7 @@ 1. For ordering: * semaphore, condition variables -* methdos: `wait()` and `notify()` +* methods: `wait()` and `notify()` --- diff --git a/content/chapters/compute/lecture/slides/synchronization.md b/chapters/compute/synchronization/slides/synchronization.md similarity index 73% rename from content/chapters/compute/lecture/slides/synchronization.md rename to chapters/compute/synchronization/slides/synchronization.md index d2ecf15c88..57ffac3964 100644 --- a/content/chapters/compute/lecture/slides/synchronization.md +++ b/chapters/compute/synchronization/slides/synchronization.md @@ -28,7 +28,7 @@ * `demo/race-condition/race_condition.c` * `var++` equivalency (**critical section**): -![Race Condition - Instructions](./media/race-condition-generated.gif) +![Race Condition - Instructions](../generated-media/race-condition-generated.gif) * In the end `var = 1` or `var = 2` @@ -38,13 +38,13 @@ ```c if (lock = 0) { - lock = 1; + lock = 1; /* critical section */ lock = 0; } ``` -![Race Condition - TOCTOU](./media/race-condition-toctou-generated.gif) +![Race Condition - TOCTOU](../generated-media/race-condition-toctou-generated.gif) * **Wrong:** threads enter critical section simultaneously @@ -54,7 +54,7 @@ if (lock = 0) { * Only allow **one thread** to access the critical section at a given time (mutual exclusion) -![Race Condition - Using a lock](./media/race-condition-lock-generated.gif) +![Race Condition - Using a lock](../generated-media/race-condition-lock-generated.gif) * In the end `var` is **always** 2 -* [Quiz](../quiz/not-race-condition.md) +* [Quiz](../drills/questions/not-race-condition.md) diff --git a/content/chapters/compute/lecture/slides/thread-safety.md b/chapters/compute/synchronization/slides/thread-safety.md similarity index 92% rename from content/chapters/compute/lecture/slides/thread-safety.md rename to chapters/compute/synchronization/slides/thread-safety.md index 112491981a..9374c408bb 100644 --- a/content/chapters/compute/lecture/slides/thread-safety.md +++ b/chapters/compute/synchronization/slides/thread-safety.md @@ -10,7 +10,7 @@ * **Reentrant function:** preempted at any time and re-called from the same thread with the same effects * Different from recursion: -![Recursion vs reentrancy](./media/recursive-vs-reentrant.svg) +![Recursion vs reentrancy](../media/recursive-vs-reentrant.svg) ---- diff --git a/content/chapters/compute/lab/quiz/seg-fault-exit-code.md b/chapters/compute/threads/drills/questions/seg-fault-exit-code.md similarity index 90% rename from content/chapters/compute/lab/quiz/seg-fault-exit-code.md rename to chapters/compute/threads/drills/questions/seg-fault-exit-code.md index 1334d6c83b..d88f2bc2e6 100644 --- a/content/chapters/compute/lab/quiz/seg-fault-exit-code.md +++ b/chapters/compute/threads/drills/questions/seg-fault-exit-code.md @@ -1,4 +1,4 @@ -# Seg Fault Exit Code +# Segfault Exit Code ## Question Text @@ -17,6 +17,6 @@ What is the exit code of the faulty child process spawned by `support/sum-array- ## Feedback We can obtain the number of the signal that killed a child process via the second argument of the `waitpid` syscall. -We can use the `WIFSIGNALED()` and `WTERMSIG()` marcros. +We can use the `WIFSIGNALED()` and `WTERMSIG()` macros. By doing so, we see the exit code of the faulty child process is 11. We can then use the `kill -l` command to view the code of each signal and `SIGSEGV` has the code 11. diff --git a/content/chapters/compute/lab/quiz/thread-memory.md b/chapters/compute/threads/drills/questions/thread-memory.md similarity index 100% rename from content/chapters/compute/lab/quiz/thread-memory.md rename to chapters/compute/threads/drills/questions/thread-memory.md diff --git a/content/chapters/compute/lecture/quiz/threads-shared-data.md b/chapters/compute/threads/drills/questions/threads-shared-data.md similarity index 100% rename from content/chapters/compute/lecture/quiz/threads-shared-data.md rename to chapters/compute/threads/drills/questions/threads-shared-data.md diff --git a/chapters/compute/threads/drills/tasks/multithreaded/.gitignore b/chapters/compute/threads/drills/tasks/multithreaded/.gitignore new file mode 100644 index 0000000000..296b2ccf64 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/.gitignore @@ -0,0 +1 @@ +support/ diff --git a/chapters/compute/threads/drills/tasks/multithreaded/Makefile b/chapters/compute/threads/drills/tasks/multithreaded/Makefile new file mode 100644 index 0000000000..12c6b032d2 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/Makefile @@ -0,0 +1,8 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + $(PYTHON) $(SCRIPT) --input ./solution/ --output ./support/ + +clean: + rm -rf support \ No newline at end of file diff --git a/chapters/compute/threads/drills/tasks/multithreaded/README.md b/chapters/compute/threads/drills/tasks/multithreaded/README.md new file mode 100644 index 0000000000..adf9a6f5a8 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/README.md @@ -0,0 +1,33 @@ +# Multithreaded + +Enter the `multithreaded/support/` folder and go through the practice items below. + +1. Use the Makefile to compile `multithread.c`, run it and follow the instructions. + + The aim of this task is to familiarize you with the `pthreads` library. + In order to use it, you have to add `#include ` in `multithreaded.c` and `-lpthread` in the compiler options. + + The executable creates 5 threads besides the main thread, puts each of them to sleep for **5 seconds**, then waits for all of them to finish. + Give it a run and notice that the total waiting time is around **5 seconds** since you started the last thread. + That is the whole point - they each run in parallel. + +1. Make each thread print its ID once it is done sleeping. + + Create a new function `sleep_wrapper2()` identical to `sleep_wrapper()` to organize your work. + So far, the `data` argument is unused (mind the `__unused` attribute), so that is your starting point. + You cannot change `sleep_wrapper2()` definition, since `pthreads_create()` expects a pointer to a function that receives a `void *` argument. + What you can and should do is to pass a pointer to a `int` as argument, and then cast `data` to `int *` inside `sleep_wrapper2()`. + + `Note:` Do not simply pass `&i` as argument to the function. + This will make all threads to use the **same integer** as their ID. + + `Note:` Do not use global variables. + + If you get stuck you can google `pthread example` and you will probably stumble upon [this](https://gist.github.com/ankurs/179778). + +1. On top of printing its ID upon completion, make each thread sleep for a different amount of time. + + Create a new function `sleep_wrapper3()` identical to `sleep_wrapper()` to organize your work. + The idea is to repeat what you did on the previous exercise and use the right argument for `sleep_wrapper3()`. + Keep in mind that you cannot change its definition. + Bonus points if you do not use the thread's ID as the sleeping amount. diff --git a/chapters/compute/threads/drills/tasks/multithreaded/generate_skels.py b/chapters/compute/threads/drills/tasks/multithreaded/generate_skels.py new file mode 100644 index 0000000000..2589f9b05e --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/generate_skels.py @@ -0,0 +1,150 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/.gitignore b/chapters/compute/threads/drills/tasks/multithreaded/solution/.gitignore new file mode 100644 index 0000000000..1b7bf69900 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/solution/.gitignore @@ -0,0 +1 @@ +multithreaded diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/Makefile b/chapters/compute/threads/drills/tasks/multithreaded/solution/Makefile new file mode 100644 index 0000000000..3d99144399 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/solution/Makefile @@ -0,0 +1,41 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +SRCS = multithreaded.c +OBJS = $(SRCS:.c=.o) +BINARIES = multithreaded + +# Default rule: Build everything +all: $(BINARIES) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile object files from source files +%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create multithreaded binary +multithreaded: multithreaded.o $(LOGGER) + $(CC) $(CFLAGS) multithreaded.o $(LOGGER) -o multithreaded $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean \ No newline at end of file diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/multithreaded.c b/chapters/compute/threads/drills/tasks/multithreaded/solution/multithreaded.c new file mode 100644 index 0000000000..c113235e88 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/solution/multithreaded.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include + +#include "./utils/utils.h" + +#define __unused __attribute_maybe_unused__ + +static void wait_for_input(const char *msg) +{ + char buffer[128]; + + printf("%s ...", msg); + fgets(buffer, 128, stdin); +} + + +static void *sleep_wrapper(void __unused * data) +{ + sleep(5); + return NULL; +} + +/* TODO 8: Define sleep_wrapper2() which should print the threadID */ +static void *sleep_wrapper2(void *data) +{ + int id = *(int *)data; + + sleep(2); + printf("Thread [%d] done\n", id); + return NULL; +} + +/* REPLACE 1 */ +/* static void *sleep_wrapper2(void __unused *data); */ + +/* TODO 13: Define sleep_wrapper3() which should print the threadID and sleep for a variable time */ +struct _thread_data { + int id; + int sleep_time; +}; + +static void *sleep_wrapper3(void *data) +{ + struct _thread_data *thread_data = (struct _thread_data *)data; + + sleep(thread_data->sleep_time); + printf("Thread [%d] done after %d seconds\n", thread_data->id, thread_data->sleep_time); + return NULL; +} + +/* REPLACE 1 */ +/* static void *sleep_wrapper3(void __unused *data); */ + +#define NUM_THREADS 5 + +int main(void) +{ + size_t i; + pthread_t tid[NUM_THREADS]; + int rc; + + /* REMOVE 8 */ + struct _thread_data thread_data[NUM_THREADS] = { + {0, 2}, + {1, 3}, + {2, 1}, + {3, 4}, + {4, 2} + }; + + /* TODO 1: Define an array of thread ids*/ + int thread_id[NUM_THREADS] = {0, 1, 2, 3, 4}; + + for (i = 0; i < NUM_THREADS; i++) { + wait_for_input("Press key to create new thread"); + /* TODO 1: Create a new thread using sleep_wrapper3() */ + rc = pthread_create(&tid[i], NULL, sleep_wrapper3, &thread_data[i]); + + /* TODO 1: Create a new thread using sleep_wrapper2() */ + rc = pthread_create(&tid[i], NULL, sleep_wrapper2, &thread_id[i]); + + /* REPLACE 1 */ + /* rc = pthread_create(&tid[i], NULL, sleep_wrapper, NULL); */ + + DIE(rc < 0, "pthread_create"); + printf("Thread [%zu] created\n", i); + } + + wait_for_input("Press key to wait for threads"); + for (i = 0; i < NUM_THREADS; i++) { + pthread_join(tid[i], NULL); + printf("Thread [%zu] joined\n", i); + } + + return 0; +} diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/log/CPPLINT.cfg b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/log/log.c b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/log/log.h b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.c b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.h b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/utils.h b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/README.md b/chapters/compute/threads/drills/tasks/sum-array-bugs/README.md new file mode 100644 index 0000000000..eae7d04cba --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array-bugs/README.md @@ -0,0 +1,48 @@ +# Wait for It + +The process that spawns all the others and subsequently calls `waitpid` to wait for them to finish can also get their return codes. +Update the code in `sum-array-bugs/support/seg-fault/sum_array_processes.c` and modify the call to `waitpid` to obtain and investigate this return code. +Display an appropriate message if one of the child processes returns an error. + +Remember to use the appropriate [macros](https://linux.die.net/man/2/waitpid) for handling the `status` variable that is modified by `waitpid()`, as it is a bit-field. +When a process runs into a system error, it receives a signal. +A signal is a means to interrupt the normal execution of a program from the outside. +It is associated with a number. +Use `kill -l` to find the full list of signals. + +[Quiz](../../questions/seg-fault-exit-code.md) + +So up to this point we've seen that one advantage of processes is that they offer better safety than threads. +Because they use separate virtual address spaces, sibling processes are better isolated than threads. +Thus, an application that uses processes can be more robust to errors than if it were using threads. + +## Memory Corruption + +Because they share the same address space, threads run the risk of corrupting each other's data. +Take a look at the code in `sum-array-bugs/support/memory-corruption/python/`. +The two programs only differ in how they spread their workload. +One uses threads while the other uses processes. + +Run both programs with and without memory corruption. +Pass any value as a third argument to trigger the corruption. + +```console +student@os:~/.../sum-array-bugs/support/memory-corruption/python$ python3 memory_corruption_processes.py # no memory corruption +[...] + +student@os:~/.../sum-array-bugs/support/memory-corruption/python$ python3 memory_corruption_processes.py 1 # do memory corruption +[...] +``` + +The one using threads will most likely print a negative sum, while the other displays the correct sum. +This happens because all threads refer to the same memory for the array `arr`. +What happens to the processes is a bit more complicated. + +[Later in this lab](../../../../copy-on-write/reading/copy-on-write.md), we will see that initially, the page tables of all processes point to the same physical frames or `arr`. +When the malicious process tries to corrupt this array by **writing data to it**, the OS duplicates the original frames of `arr` so that the malicious process writes the corrupted values to these new frames, while leaving the original ones untouched. +This mechanism is called **Copy-on-Write** and is an OS optimisation so that memory is shared between the parent and the child process, until one of them attempts to write to it. +At this point, this process receives its own separate copies of the previously shared frames. + +Note that in order for the processes to share the `sums` dictionary, it is not created as a regular dictionary, but using the `Manager` module. +This module provides some special data structures that are allocated in **shared memory** so that all processes can access them. +You can learn more about shared memory and its various implementations [in the Arena section](shared-memory). diff --git a/content/chapters/compute/lab/solution/sum-array-bugs/seg-fault/sum_array_processes.d b/chapters/compute/threads/drills/tasks/sum-array-bugs/solution/seg-fault/sum_array_processes.d similarity index 100% rename from content/chapters/compute/lab/solution/sum-array-bugs/seg-fault/sum_array_processes.d rename to chapters/compute/threads/drills/tasks/sum-array-bugs/solution/seg-fault/sum_array_processes.d diff --git a/content/chapters/compute/lab/support/sum-array-bugs/memory-corruption/python/memory_corruption_processes.py b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/memory-corruption/python/memory_corruption_processes.py similarity index 100% rename from content/chapters/compute/lab/support/sum-array-bugs/memory-corruption/python/memory_corruption_processes.py rename to chapters/compute/threads/drills/tasks/sum-array-bugs/support/memory-corruption/python/memory_corruption_processes.py diff --git a/content/chapters/compute/lab/support/sum-array-bugs/memory-corruption/python/memory_corruption_threads.py b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/memory-corruption/python/memory_corruption_threads.py similarity index 100% rename from content/chapters/compute/lab/support/sum-array-bugs/memory-corruption/python/memory_corruption_threads.py rename to chapters/compute/threads/drills/tasks/sum-array-bugs/support/memory-corruption/python/memory_corruption_threads.py diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/.gitignore b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/.gitignore new file mode 100644 index 0000000000..cffe4b4994 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/.gitignore @@ -0,0 +1,2 @@ +sum_array_threads +sum_array_processes diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/Makefile b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/Makefile new file mode 100644 index 0000000000..8c8f0785e9 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/Makefile @@ -0,0 +1,43 @@ +BINARIES = sum_array_threads sum_array_processes + +all: $(BINARIES) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARIES): $(LOGGER) + +clean:: + -rm -f $(BINARIES) + +.PHONY: all clean + +LDLIBS += -lpthread + +sum_array_threads: sum_array_threads.o generate_random_array.o + +sum_array_processes: sum_array_processes.o generate_random_array.o diff --git a/content/chapters/compute/lab/support/sum-array/c/generate_random_array.c b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/generate_random_array.c similarity index 100% rename from content/chapters/compute/lab/support/sum-array/c/generate_random_array.c rename to chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/generate_random_array.c diff --git a/content/chapters/compute/lab/support/sum-array-bugs/seg-fault/include/array_utils.h b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/include/array_utils.h similarity index 100% rename from content/chapters/compute/lab/support/sum-array-bugs/seg-fault/include/array_utils.h rename to chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/include/array_utils.h diff --git a/content/chapters/compute/lab/support/sum-array/c/include/generate_random_array.h b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/include/generate_random_array.h similarity index 100% rename from content/chapters/compute/lab/support/sum-array/c/include/generate_random_array.h rename to chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/include/generate_random_array.h diff --git a/content/chapters/compute/lab/support/sum-array-bugs/seg-fault/sum_array_processes.c b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/sum_array_processes.c similarity index 100% rename from content/chapters/compute/lab/support/sum-array-bugs/seg-fault/sum_array_processes.c rename to chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/sum_array_processes.c diff --git a/content/chapters/compute/lab/support/sum-array-bugs/seg-fault/sum_array_threads.c b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/sum_array_threads.c similarity index 100% rename from content/chapters/compute/lab/support/sum-array-bugs/seg-fault/sum_array_threads.c rename to chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/sum_array_threads.c diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/log/CPPLINT.cfg b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/log/log.c b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/log/log.h b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.c b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.h b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/utils.h b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/threads/drills/tasks/sum-array/README.md b/chapters/compute/threads/drills/tasks/sum-array/README.md new file mode 100644 index 0000000000..d7500bcf6c --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/README.md @@ -0,0 +1,48 @@ +# Libraries for Parallel Processing + +In `sum-array/support/c/sum_array_threads.c` we spawned threads "manually" by using the `pthread_create()` function. +This is **not** a syscall, but a wrapper over the common syscall used by both `fork()` (which is also not a syscall) and `pthread_create()`. + +Still, `pthread_create()` is not yet a syscall. +In order to see what syscall `pthread_create()` uses, check out [this section](./guides/clone.md). + +Most programming languages provide a more advanced API for handling parallel computation. + +## `std.parallelism` in D + +D language's standard library exposes the [`std.parallelism`](https://dlang.org/phobos/std_parallelism.html), which provides a series of parallel processing functions. +One such function is `reduce()`, which splits an array between a given number of threads and applies a given operation to these chunks. +In our case, the operation simply adds the elements to an accumulator: `a + b`. +Follow and run the code in `sum-array/support/d/sum_array_threads_reduce.d`. + +The number of threads is used within a [`TaskPool`](https://dlang.org/phobos/std_parallelism.html#.TaskPool). +This structure is a thread manager (not scheduler). +It silently creates the number of threads we request and then `reduce()` spreads its workload between these threads. + +## OpenMP for C + +Unlike D, C does not support parallel computation by design. +It needs a library to do advanced things, like `reduce()` from D. +We have chosen to use the OpenMP library for this. +Follow the code in `support/sum-array/c/sum_array_threads_openmp.c`. + +The `#pragma` used in the code instructs the compiler to enable the `omp` module, and to parallelise the code. +In this case, we instruct the compiler to perform a reduce of the array, using the `+` operator, and to store the results in the `result` variable. +This reduction uses threads to calculate the sum, similar to `summ_array_threads.c`, but in a much more optimised form. + +Now compile and run the `sum_array_threads_openmp` binary using 1, 2, 4, and 8 threads as before. +You'll see lower running times than `sum_array_threads` due to the highly-optimised code emitted by the compiler. +For this reason and because library functions are usually much better tested than your own code, it is always preferred to use a library function for a given task. + +## Array Sum in Python + +Let's first probe this by implementing two parallel versions of the code in `sum-array/support/python/sum_array_sequential.py`. +One version should use threads and the other should use processes. +Run each of them using 1, 2, 4, and 8 threads / processes respectively and compare the running times. +Notice that the running times of the multithreaded implementation do not decrease. +This is because the GIL makes it so that those threads that you create essentially run sequentially. + +The GIL also makes it so that individual Python instructions are atomic. +Run the code in `race-condition/support/python/race_condition.py`. +Every time, `var` will be 0 because the GIL doesn't allow the two threads to run in parallel and reach the critical section at the same time. +This means that the instructions `var += 1` and `var -= 1` become atomic. diff --git a/content/chapters/compute/lab/solution/sum-array/python/sum_array_processes.py b/chapters/compute/threads/drills/tasks/sum-array/solution/python/sum_array_processes.py similarity index 100% rename from content/chapters/compute/lab/solution/sum-array/python/sum_array_processes.py rename to chapters/compute/threads/drills/tasks/sum-array/solution/python/sum_array_processes.py diff --git a/content/chapters/compute/lab/solution/sum-array/python/sum_array_threads.py b/chapters/compute/threads/drills/tasks/sum-array/solution/python/sum_array_threads.py similarity index 100% rename from content/chapters/compute/lab/solution/sum-array/python/sum_array_threads.py rename to chapters/compute/threads/drills/tasks/sum-array/solution/python/sum_array_threads.py diff --git a/content/chapters/compute/lab/.gitignore b/chapters/compute/threads/drills/tasks/sum-array/support/c/.gitignore similarity index 70% rename from content/chapters/compute/lab/.gitignore rename to chapters/compute/threads/drills/tasks/sum-array/support/c/.gitignore index 4a234ffb5b..d57760f9b1 100644 --- a/content/chapters/compute/lab/.gitignore +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/.gitignore @@ -1,4 +1,4 @@ +sum_array_threads_openmp +sum_array_processes sum_array_sequential sum_array_threads -sum_array_processes -sum_array_threads_reduce diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/Makefile b/chapters/compute/threads/drills/tasks/sum-array/support/c/Makefile new file mode 100644 index 0000000000..55db62eb03 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/Makefile @@ -0,0 +1,47 @@ +BINARIES = sum_array_sequential sum_array_threads sum_array_processes sum_array_threads_openmp +CFLAGS += -fopenmp +LDFLAGS += -fopenmp + +all: $(BINARIES) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARIES): $(LOGGER) + +clean:: + -rm -f $(BINARIES) + +.PHONY: all clean + +sum_array_sequential: sum_array_sequential.o generate_random_array.o + +sum_array_threads: sum_array_threads.o generate_random_array.o + +sum_array_processes: sum_array_processes.o generate_random_array.o + +sum_array_threads_openmp: sum_array_threads_openmp.o generate_random_array.o diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/generate_random_array.c b/chapters/compute/threads/drills/tasks/sum-array/support/c/generate_random_array.c new file mode 100644 index 0000000000..f937d544b0 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/generate_random_array.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include "include/generate_random_array.h" + +void generate_random_array(size_t length, int array[]) +{ + uint32_t seed = time(0); + + for (size_t i = 0; i < length; i++) + array[i] = rand_r(&seed) % MAX_ELEMENT; +} diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/include/array_utils.h b/chapters/compute/threads/drills/tasks/sum-array/support/c/include/array_utils.h new file mode 100644 index 0000000000..3b171f3cfb --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/include/array_utils.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef ARRAY_UTILS_H +#define ARRAY_UTILS_H + +#define ARR_LEN 100000000 + +#define min(a, b) \ + ({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + _a < _b ? _a : _b; \ + }) + +#endif /* ARRAY_UTILS_H */ diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/include/generate_random_array.h b/chapters/compute/threads/drills/tasks/sum-array/support/c/include/generate_random_array.h new file mode 100644 index 0000000000..0773106854 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/include/generate_random_array.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef GENERATE_RANDOM_ARRAY +#define GENERATE_RANDOM_ARRAY + +#define MAX_ELEMENT 1000 + +void generate_random_array(size_t length, int array[]); + +#endif /* GENERATE_RANDOM_ARRAY */ diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_processes.c b/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_processes.c new file mode 100644 index 0000000000..0e06da80b5 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_processes.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "include/generate_random_array.h" +#include "include/array_utils.h" +#include "utils/utils.h" + +static void calculate_array_part_sum(int *array, size_t start, size_t end, + long *results, size_t tid) +{ + long sum_array = 0; + + for (size_t i = start; i < end; ++i) + sum_array += array[i]; + + results[tid] = sum_array; +} + +static inline void *create_shared_results_array(size_t num_processes) +{ + /* + * This buffer will be shared by the process that creates it and all its + * children due to the `MAP_SHARED` argument + */ + return mmap(NULL, num_processes * sizeof(long), + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0); +} + +int main(int argc, char *argv[]) +{ + int num_processes, ret = 0; + struct timeval start, stop; + long time; + pid_t *children; + pid_t pidret; + long *results; + long final_result = 0; + int *array; + + if (argc < 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(1); + } + + num_processes = atoi(argv[1]); + results = (long *)create_shared_results_array(num_processes); + DIE(!results, "Error when mapping results array"); + + array = malloc(sizeof(*array) * ARR_LEN); + if (!array) { + fprintf(stderr, "Error when allocating array: %s\n", strerror(errno)); + ret = errno; + goto error_malloc_array; + } + + children = malloc(sizeof(*children) * num_processes); + if (!children) { + fprintf(stderr, "Error when allocating children: %s\n", strerror(errno)); + ret = errno; + goto error_malloc_children; + } + + generate_random_array(ARR_LEN, array); + + gettimeofday(&start, NULL); + + for (int i = 0; i < num_processes; ++i) { + size_t elems_per_process = ARR_LEN / num_processes; + size_t start = i * elems_per_process; + size_t end = min((size_t)ARR_LEN, (i + 1) * elems_per_process); + + /* Spawn another process with the current one as its parent. */ + children[i] = fork(); + switch (children[i]) { + case -1: + ret = 1; + goto end; + case 0: + calculate_array_part_sum(array, start, end, results, i); + goto end; + default: + continue; + } + } + + for (int i = 0; i < num_processes; ++i) { + pidret = waitpid(children[i], NULL, 0); + if (pidret < 0) { + fprintf(stderr, "waitpid ended with error for process %d\n", i); + continue; + } + final_result += results[i]; + } + + gettimeofday(&stop, NULL); + + time = 1000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) / 1000; + + printf("Array sum is %ld\nTime spent: %lu ms\n", final_result, time); + +end: + free(children); + +error_malloc_children: + free(array); + +error_malloc_array: + munmap(results, num_processes * sizeof(*results)); + + return ret; +} diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_sequential.c b/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_sequential.c new file mode 100644 index 0000000000..f65bb20ece --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_sequential.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include + +#include "include/generate_random_array.h" +#include "include/array_utils.h" +#include "utils/utils.h" + +int main(void) +{ + long result = 0; + struct timeval start, stop; + long time; + int *array; + + array = malloc(sizeof(*array) * ARR_LEN); + DIE(!array, "Error when allocating array"); + + generate_random_array(ARR_LEN, array); + + gettimeofday(&start, NULL); + + for (int i = 0; i < ARR_LEN; i++) + result += array[i]; + + gettimeofday(&stop, NULL); + + time = 1000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) / 1000; + + printf("Array sum is %ld\nTime spent: %lu miliseconds\n", result, time); + + free(array); + + return 0; +} diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_threads.c b/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_threads.c new file mode 100644 index 0000000000..e742e78964 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_threads.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include + +#include "include/generate_random_array.h" +#include "include/array_utils.h" +#include "utils/utils.h" + +struct thread_args { + int *array; + size_t start; + size_t end; + long *results; + size_t tid; +}; + +static void *calculate_array_part_sum(void *varg) +{ + struct thread_args *arg = (struct thread_args *)varg; + long sum_array = 0; + + for (size_t i = arg->start; i < arg->end; i++) + sum_array += arg->array[i]; + + arg->results[arg->tid] = sum_array; + + return NULL; +} + +int main(int argc, char *argv[]) +{ + int num_threads; + struct timeval start, stop; + long time; + pthread_t *threads; + long *results; + struct thread_args *args; + int ret; + long final_result = 0; + int *array; + + if (argc < 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(1); + } + + num_threads = atoi(argv[1]); + + array = malloc(sizeof(*array) * ARR_LEN); + DIE(!array, "Error when allocating array"); + + threads = malloc(sizeof(*threads) * num_threads); + if (!threads) { + fprintf(stderr, "Error when allocating threads: %s", strerror(errno)); + ret = errno; + goto error_malloc_threads; + } + + results = malloc(sizeof(*results) * num_threads); + if (!results) { + fprintf(stderr, "Error when allocating results: %s", strerror(errno)); + ret = errno; + goto error_malloc_results; + } + + args = malloc(sizeof(*args) * num_threads); + if (!args) { + fprintf(stderr, "Error when allocating thread args: %s", strerror(errno)); + ret = errno; + goto error_malloc_args; + } + + generate_random_array(ARR_LEN, array); + + gettimeofday(&start, NULL); + + for (int i = 0; i < num_threads; i++) { + size_t elems_per_thread = ARR_LEN / num_threads; + size_t start = i * elems_per_thread; + size_t end = min((size_t)ARR_LEN, (i + 1) * elems_per_thread); + + args[i].array = array; + args[i].start = start; + args[i].end = end; + args[i].results = results; + args[i].tid = i; + + ret = pthread_create(&threads[i], NULL, calculate_array_part_sum, (void *)&args[i]); + if (ret != 0) { + fprintf(stderr, "Error when creating thread #%d: %s", i, strerror(errno)); + goto error_pthread_create; + } + } + + for (int i = 0; i < num_threads; i++) + pthread_join(threads[i], NULL); + + for (int i = 0; i < num_threads; i++) + final_result += results[i]; + + gettimeofday(&stop, NULL); + + time = 1000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) / 1000; + + printf("Array sum is %ld\nTime spent: %lu ms\n", final_result, time); + +error_pthread_create: + free(args); + +error_malloc_args: + free(results); + +error_malloc_results: + free(threads); + +error_malloc_threads: + free(array); + + return ret; +} diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_threads_openmp.c b/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_threads_openmp.c new file mode 100644 index 0000000000..aec15f30a7 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/sum_array_threads_openmp.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include + +#include "include/generate_random_array.h" +#include "include/array_utils.h" +#include "utils/utils.h" + +int main(int argc, char *argv[]) +{ + struct timeval start, stop; + long time, result = 0; + int num_threads; + int *array; + + if (argc < 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(1); + } + + array = malloc(sizeof(*array) * ARR_LEN); + DIE(!array, "Error when allocating array"); + + generate_random_array(ARR_LEN, array); + + num_threads = atoi(argv[1]); + + omp_set_num_threads(num_threads); + + gettimeofday(&start, NULL); + +#pragma omp parallel for reduction(+ \ + : result) + for (int i = 0; i < ARR_LEN; i++) + result += array[i]; + + gettimeofday(&stop, NULL); + + time = 1000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) / 1000; + + printf("Array sum is %ld\nTime spent: %lu ms\n", result, time); + + free(array); + + return 0; +} diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/log/CPPLINT.cfg b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/log/log.c b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/log/log.h b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.c b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.h b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/utils.h b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/support/sum-array/d/Makefile b/chapters/compute/threads/drills/tasks/sum-array/support/d/Makefile similarity index 100% rename from content/chapters/compute/lab/support/sum-array/d/Makefile rename to chapters/compute/threads/drills/tasks/sum-array/support/d/Makefile diff --git a/content/chapters/compute/lab/support/sum-array/d/generate_random_array.d b/chapters/compute/threads/drills/tasks/sum-array/support/d/generate_random_array.d similarity index 100% rename from content/chapters/compute/lab/support/sum-array/d/generate_random_array.d rename to chapters/compute/threads/drills/tasks/sum-array/support/d/generate_random_array.d diff --git a/content/chapters/compute/lab/support/sum-array/d/sum_array_threads_reduce.d b/chapters/compute/threads/drills/tasks/sum-array/support/d/sum_array_threads_reduce.d similarity index 100% rename from content/chapters/compute/lab/support/sum-array/d/sum_array_threads_reduce.d rename to chapters/compute/threads/drills/tasks/sum-array/support/d/sum_array_threads_reduce.d diff --git a/content/chapters/compute/lab/support/sum-array/python/sum_array_sequential.py b/chapters/compute/threads/drills/tasks/sum-array/support/python/sum_array_sequential.py similarity index 100% rename from content/chapters/compute/lab/support/sum-array/python/sum_array_sequential.py rename to chapters/compute/threads/drills/tasks/sum-array/support/python/sum_array_sequential.py diff --git a/chapters/compute/threads/guides/clone/README.md b/chapters/compute/threads/guides/clone/README.md new file mode 100644 index 0000000000..9dde8c33a4 --- /dev/null +++ b/chapters/compute/threads/guides/clone/README.md @@ -0,0 +1,28 @@ +# Threads and Processes: `clone` + +Let's go back to our initial demos that used threads and processes. +We'll see that in order to create both threads and processes, the underlying Linux syscall is `clone`. +For this, we'll run both `sum_array_threads` and `sum_array_processes` under `strace`. +As we've already established, we're only interested in the `clone` syscall: + +```console +student@os:~/.../sum-array/support/c$ strace -e clone,clone3 ./sum_array_threads 2 +clone(child_stack=0x7f60b56482b0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[1819693], tls=0x7f60b5649640, child_tidptr=0x7f60b5649910) = 1819693 +clone(child_stack=0x7f60b4e472b0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[1819694], tls=0x7f60b4e48640, child_tidptr=0x7f60b4e48910) = 1819694 + +student@os:~/.../sum-array/support/c$ strace -e clone,clone3 ./sum_array_processes 2 +clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f7a4e346650) = 1820599 +clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f7a4e346650) = 1820600 +``` + +We ran each program with an argument of 2, so we have 2 calls to `clone`. +Notice that in the case of threads, the `clone3 syscall receives more arguments. +The relevant flags passed as arguments when creating threads are documented in [`clone`'s man page](): + +- `CLONE_VM`: the child and the parent process share the same VAS +- `CLONE_{FS,FILES,SIGHAND}`: the new thread shares the filesystem information, file and signal handlers with the one that created it +The syscall also receives valid pointers to the new thread's stack and TLS, i.e. the only parts of the VAS that are distinct between threads (although they are technically accessible from all threads). + +By contrast, when creating a new process, the arguments of the `clone` syscall are simpler (i.e. fewer flags are present). +Remember that in both cases `clone` creates a new **thread**. +When creating a process, `clone` creates this new thread within a new separate address space. diff --git a/chapters/compute/threads/guides/clone/support/Makefile b/chapters/compute/threads/guides/clone/support/Makefile new file mode 100644 index 0000000000..84d2419457 --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/Makefile @@ -0,0 +1,43 @@ +BINARIES = sum_array_threads sum_array_processes +CFLAGS += -fopenmp +LDFLAGS += -fopenmp + +all: $(BINARIES) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARIES): $(LOGGER) + +clean:: + -rm -f $(BINARIES) + +.PHONY: all clean + +sum_array_threads: sum_array_threads.o generate_random_array.o + +sum_array_processes: sum_array_processes.o generate_random_array.o diff --git a/chapters/compute/threads/guides/clone/support/generate_random_array.c b/chapters/compute/threads/guides/clone/support/generate_random_array.c new file mode 100644 index 0000000000..f937d544b0 --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/generate_random_array.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include "include/generate_random_array.h" + +void generate_random_array(size_t length, int array[]) +{ + uint32_t seed = time(0); + + for (size_t i = 0; i < length; i++) + array[i] = rand_r(&seed) % MAX_ELEMENT; +} diff --git a/chapters/compute/threads/guides/clone/support/include/array_utils.h b/chapters/compute/threads/guides/clone/support/include/array_utils.h new file mode 100644 index 0000000000..3b171f3cfb --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/include/array_utils.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef ARRAY_UTILS_H +#define ARRAY_UTILS_H + +#define ARR_LEN 100000000 + +#define min(a, b) \ + ({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + _a < _b ? _a : _b; \ + }) + +#endif /* ARRAY_UTILS_H */ diff --git a/chapters/compute/threads/guides/clone/support/include/generate_random_array.h b/chapters/compute/threads/guides/clone/support/include/generate_random_array.h new file mode 100644 index 0000000000..0773106854 --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/include/generate_random_array.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef GENERATE_RANDOM_ARRAY +#define GENERATE_RANDOM_ARRAY + +#define MAX_ELEMENT 1000 + +void generate_random_array(size_t length, int array[]); + +#endif /* GENERATE_RANDOM_ARRAY */ diff --git a/chapters/compute/threads/guides/clone/support/sum_array_processes.c b/chapters/compute/threads/guides/clone/support/sum_array_processes.c new file mode 100644 index 0000000000..0e06da80b5 --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/sum_array_processes.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "include/generate_random_array.h" +#include "include/array_utils.h" +#include "utils/utils.h" + +static void calculate_array_part_sum(int *array, size_t start, size_t end, + long *results, size_t tid) +{ + long sum_array = 0; + + for (size_t i = start; i < end; ++i) + sum_array += array[i]; + + results[tid] = sum_array; +} + +static inline void *create_shared_results_array(size_t num_processes) +{ + /* + * This buffer will be shared by the process that creates it and all its + * children due to the `MAP_SHARED` argument + */ + return mmap(NULL, num_processes * sizeof(long), + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0); +} + +int main(int argc, char *argv[]) +{ + int num_processes, ret = 0; + struct timeval start, stop; + long time; + pid_t *children; + pid_t pidret; + long *results; + long final_result = 0; + int *array; + + if (argc < 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(1); + } + + num_processes = atoi(argv[1]); + results = (long *)create_shared_results_array(num_processes); + DIE(!results, "Error when mapping results array"); + + array = malloc(sizeof(*array) * ARR_LEN); + if (!array) { + fprintf(stderr, "Error when allocating array: %s\n", strerror(errno)); + ret = errno; + goto error_malloc_array; + } + + children = malloc(sizeof(*children) * num_processes); + if (!children) { + fprintf(stderr, "Error when allocating children: %s\n", strerror(errno)); + ret = errno; + goto error_malloc_children; + } + + generate_random_array(ARR_LEN, array); + + gettimeofday(&start, NULL); + + for (int i = 0; i < num_processes; ++i) { + size_t elems_per_process = ARR_LEN / num_processes; + size_t start = i * elems_per_process; + size_t end = min((size_t)ARR_LEN, (i + 1) * elems_per_process); + + /* Spawn another process with the current one as its parent. */ + children[i] = fork(); + switch (children[i]) { + case -1: + ret = 1; + goto end; + case 0: + calculate_array_part_sum(array, start, end, results, i); + goto end; + default: + continue; + } + } + + for (int i = 0; i < num_processes; ++i) { + pidret = waitpid(children[i], NULL, 0); + if (pidret < 0) { + fprintf(stderr, "waitpid ended with error for process %d\n", i); + continue; + } + final_result += results[i]; + } + + gettimeofday(&stop, NULL); + + time = 1000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) / 1000; + + printf("Array sum is %ld\nTime spent: %lu ms\n", final_result, time); + +end: + free(children); + +error_malloc_children: + free(array); + +error_malloc_array: + munmap(results, num_processes * sizeof(*results)); + + return ret; +} diff --git a/chapters/compute/threads/guides/clone/support/sum_array_threads.c b/chapters/compute/threads/guides/clone/support/sum_array_threads.c new file mode 100644 index 0000000000..e742e78964 --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/sum_array_threads.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include + +#include "include/generate_random_array.h" +#include "include/array_utils.h" +#include "utils/utils.h" + +struct thread_args { + int *array; + size_t start; + size_t end; + long *results; + size_t tid; +}; + +static void *calculate_array_part_sum(void *varg) +{ + struct thread_args *arg = (struct thread_args *)varg; + long sum_array = 0; + + for (size_t i = arg->start; i < arg->end; i++) + sum_array += arg->array[i]; + + arg->results[arg->tid] = sum_array; + + return NULL; +} + +int main(int argc, char *argv[]) +{ + int num_threads; + struct timeval start, stop; + long time; + pthread_t *threads; + long *results; + struct thread_args *args; + int ret; + long final_result = 0; + int *array; + + if (argc < 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(1); + } + + num_threads = atoi(argv[1]); + + array = malloc(sizeof(*array) * ARR_LEN); + DIE(!array, "Error when allocating array"); + + threads = malloc(sizeof(*threads) * num_threads); + if (!threads) { + fprintf(stderr, "Error when allocating threads: %s", strerror(errno)); + ret = errno; + goto error_malloc_threads; + } + + results = malloc(sizeof(*results) * num_threads); + if (!results) { + fprintf(stderr, "Error when allocating results: %s", strerror(errno)); + ret = errno; + goto error_malloc_results; + } + + args = malloc(sizeof(*args) * num_threads); + if (!args) { + fprintf(stderr, "Error when allocating thread args: %s", strerror(errno)); + ret = errno; + goto error_malloc_args; + } + + generate_random_array(ARR_LEN, array); + + gettimeofday(&start, NULL); + + for (int i = 0; i < num_threads; i++) { + size_t elems_per_thread = ARR_LEN / num_threads; + size_t start = i * elems_per_thread; + size_t end = min((size_t)ARR_LEN, (i + 1) * elems_per_thread); + + args[i].array = array; + args[i].start = start; + args[i].end = end; + args[i].results = results; + args[i].tid = i; + + ret = pthread_create(&threads[i], NULL, calculate_array_part_sum, (void *)&args[i]); + if (ret != 0) { + fprintf(stderr, "Error when creating thread #%d: %s", i, strerror(errno)); + goto error_pthread_create; + } + } + + for (int i = 0; i < num_threads; i++) + pthread_join(threads[i], NULL); + + for (int i = 0; i < num_threads; i++) + final_result += results[i]; + + gettimeofday(&stop, NULL); + + time = 1000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) / 1000; + + printf("Array sum is %ld\nTime spent: %lu ms\n", final_result, time); + +error_pthread_create: + free(args); + +error_malloc_args: + free(results); + +error_malloc_results: + free(threads); + +error_malloc_threads: + free(array); + + return ret; +} diff --git a/chapters/compute/threads/guides/clone/support/utils/log/CPPLINT.cfg b/chapters/compute/threads/guides/clone/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/threads/guides/clone/support/utils/log/log.c b/chapters/compute/threads/guides/clone/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/threads/guides/clone/support/utils/log/log.h b/chapters/compute/threads/guides/clone/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.c b/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.h b/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/threads/guides/clone/support/utils/utils.h b/chapters/compute/threads/guides/clone/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/threads/guides/clone/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/threads/guides/python-gil/README.md b/chapters/compute/threads/guides/python-gil/README.md new file mode 100644 index 0000000000..2f53c6b55a --- /dev/null +++ b/chapters/compute/threads/guides/python-gil/README.md @@ -0,0 +1,39 @@ +# The GIL + +Throughout this lab, you might have noticed that there were no thread exercises _in Python_. +If you did, you probably wondered why. +It's not because Python does not support threads, because it does, but because of a mechanism called the **Global Interpreter Lock**, or GIL. +As its name suggests, this is a lock implemented inside most commonly used Python interpreter (CPython), which only allows **one** thread to run at a given time. +As a consequence, multithreaded programs written in Python run **concurrently**, not in parallel. +For this reason, you will see no speedup even when you run an embarrassingly parallel code in parallel. + +However, keep in mind that this drawback does not make threads useless in Python. +They are still useful and widely used when a process needs to perform many IO-bound tasks (i.e.: tasks that involve many file reads / writes or network requests). +Such tasks run many blocking syscalls that require the thread to switch from the RUNNING state to WAITING. +Doing so voluntarily makes threads viable because they rarely run for their entire time slice and spend most of the time waiting for data. +So it doesn't hurt them to run concurrently, instead of in parallel. + +Do not make the confusion to believe threads in Python are [user-level threads](../../scheduling/reading/scheduling.md#user-level-vs-kernel-level-threads). +[`threading.Thread`](https://docs.python.org/3/library/threading.html#threading.Thread)s are kernel-level threads. +It's just that they are forced to run concurrently by the GIL. + +## But Why? + +Unlike Bigfoot, or the Loch Ness monster, we have proof that the GIL is real. +At first glance, this seems like a huge disadvantage. +Why force threads to run sequentially? +The answer has to do with memory management. +In the [Data chapter](../../../data), you learned that one way of managing memory is via _garbage collection_ (GC). +In Python, the GC uses reference counting, i.e. each object also stores the number of live pointers to it (variables that reference it). +You can see that this number needs to be modified atomically by the interpreter to avoid race conditions. +This involves adding locks to **all** Python data structures. +This large number of locks doesn't scale for a language as large and open as Python. +In addition, it also introduces the risk of _deadlocks_. +You can read more on this topic [in this article](https://realpython.com/python-gil/) and if you think eliminating the GIL looks like an easy task, which should have been done long ago, check the requirements [here](https://wiki.python.org/moin/GlobalInterpreterLock). +They're not trivial to meet. + +Single-threaded-ness is a common trope for interpreted languages to use some sort of GIL. +[Ruby MRI, the reference Ruby interpreter](https://git.ruby-lang.org/ruby.git), uses a similar mechanism, called the [Global VM Lock](https://ivoanjo.me/blog/2022/07/17/tracing-ruby-global-vm-lock/). +JavaScript is even more straightforward: it is single-threaded by design, also for GC-related reasons. +It does, however, support asynchronous actions, but these are executed on the same thread as every other code. +This is implemented by placing each instruction on a [call stack](https://medium.com/swlh/what-does-it-mean-by-javascript-is-single-threaded-language-f4130645d8a9). diff --git a/chapters/compute/threads/guides/sum-array-threads/README.md b/chapters/compute/threads/guides/sum-array-threads/README.md new file mode 100644 index 0000000000..61551eb2df --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/README.md @@ -0,0 +1,11 @@ +# Sum array Threads + +## Spreading the Work Among Other Threads + +Compile the code in `sum-array/support/c/sum_array_threads.c` and run it using 1, 2, 4 and 8 threads as you did before. +Each thread runs the `calculate_array_part_sum()` function and then finishes. +Running times should be _slightly_ smaller than the implementation using processes. +This slight time difference is caused by process creation actions, which are costlier than thread creation actions. +Because a process needs a separate virtual address space (VAS) and needs to duplicate some internal structures such as the file descriptor table and page table, it takes the operating system more time to create it than to create a thread. +On the other hand, threads belonging to the same process share the same VAS and, implicitly, the same OS-internal structures. +Therefore, they are more lightweight than processes. diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/.gitignore b/chapters/compute/threads/guides/sum-array-threads/support/c/.gitignore new file mode 100644 index 0000000000..c8e88839fc --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/.gitignore @@ -0,0 +1 @@ +sum_array_threads_openmp diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/Makefile b/chapters/compute/threads/guides/sum-array-threads/support/c/Makefile new file mode 100644 index 0000000000..55db62eb03 --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/Makefile @@ -0,0 +1,47 @@ +BINARIES = sum_array_sequential sum_array_threads sum_array_processes sum_array_threads_openmp +CFLAGS += -fopenmp +LDFLAGS += -fopenmp + +all: $(BINARIES) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARIES): $(LOGGER) + +clean:: + -rm -f $(BINARIES) + +.PHONY: all clean + +sum_array_sequential: sum_array_sequential.o generate_random_array.o + +sum_array_threads: sum_array_threads.o generate_random_array.o + +sum_array_processes: sum_array_processes.o generate_random_array.o + +sum_array_threads_openmp: sum_array_threads_openmp.o generate_random_array.o diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/generate_random_array.c b/chapters/compute/threads/guides/sum-array-threads/support/c/generate_random_array.c new file mode 100644 index 0000000000..f937d544b0 --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/generate_random_array.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include "include/generate_random_array.h" + +void generate_random_array(size_t length, int array[]) +{ + uint32_t seed = time(0); + + for (size_t i = 0; i < length; i++) + array[i] = rand_r(&seed) % MAX_ELEMENT; +} diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/include/array_utils.h b/chapters/compute/threads/guides/sum-array-threads/support/c/include/array_utils.h new file mode 100644 index 0000000000..3b171f3cfb --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/include/array_utils.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef ARRAY_UTILS_H +#define ARRAY_UTILS_H + +#define ARR_LEN 100000000 + +#define min(a, b) \ + ({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + _a < _b ? _a : _b; \ + }) + +#endif /* ARRAY_UTILS_H */ diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/include/generate_random_array.h b/chapters/compute/threads/guides/sum-array-threads/support/c/include/generate_random_array.h new file mode 100644 index 0000000000..0773106854 --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/include/generate_random_array.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef GENERATE_RANDOM_ARRAY +#define GENERATE_RANDOM_ARRAY + +#define MAX_ELEMENT 1000 + +void generate_random_array(size_t length, int array[]); + +#endif /* GENERATE_RANDOM_ARRAY */ diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_processes.c b/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_processes.c new file mode 100644 index 0000000000..0e06da80b5 --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_processes.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "include/generate_random_array.h" +#include "include/array_utils.h" +#include "utils/utils.h" + +static void calculate_array_part_sum(int *array, size_t start, size_t end, + long *results, size_t tid) +{ + long sum_array = 0; + + for (size_t i = start; i < end; ++i) + sum_array += array[i]; + + results[tid] = sum_array; +} + +static inline void *create_shared_results_array(size_t num_processes) +{ + /* + * This buffer will be shared by the process that creates it and all its + * children due to the `MAP_SHARED` argument + */ + return mmap(NULL, num_processes * sizeof(long), + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0); +} + +int main(int argc, char *argv[]) +{ + int num_processes, ret = 0; + struct timeval start, stop; + long time; + pid_t *children; + pid_t pidret; + long *results; + long final_result = 0; + int *array; + + if (argc < 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(1); + } + + num_processes = atoi(argv[1]); + results = (long *)create_shared_results_array(num_processes); + DIE(!results, "Error when mapping results array"); + + array = malloc(sizeof(*array) * ARR_LEN); + if (!array) { + fprintf(stderr, "Error when allocating array: %s\n", strerror(errno)); + ret = errno; + goto error_malloc_array; + } + + children = malloc(sizeof(*children) * num_processes); + if (!children) { + fprintf(stderr, "Error when allocating children: %s\n", strerror(errno)); + ret = errno; + goto error_malloc_children; + } + + generate_random_array(ARR_LEN, array); + + gettimeofday(&start, NULL); + + for (int i = 0; i < num_processes; ++i) { + size_t elems_per_process = ARR_LEN / num_processes; + size_t start = i * elems_per_process; + size_t end = min((size_t)ARR_LEN, (i + 1) * elems_per_process); + + /* Spawn another process with the current one as its parent. */ + children[i] = fork(); + switch (children[i]) { + case -1: + ret = 1; + goto end; + case 0: + calculate_array_part_sum(array, start, end, results, i); + goto end; + default: + continue; + } + } + + for (int i = 0; i < num_processes; ++i) { + pidret = waitpid(children[i], NULL, 0); + if (pidret < 0) { + fprintf(stderr, "waitpid ended with error for process %d\n", i); + continue; + } + final_result += results[i]; + } + + gettimeofday(&stop, NULL); + + time = 1000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) / 1000; + + printf("Array sum is %ld\nTime spent: %lu ms\n", final_result, time); + +end: + free(children); + +error_malloc_children: + free(array); + +error_malloc_array: + munmap(results, num_processes * sizeof(*results)); + + return ret; +} diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_sequential.c b/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_sequential.c new file mode 100644 index 0000000000..f65bb20ece --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_sequential.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include + +#include "include/generate_random_array.h" +#include "include/array_utils.h" +#include "utils/utils.h" + +int main(void) +{ + long result = 0; + struct timeval start, stop; + long time; + int *array; + + array = malloc(sizeof(*array) * ARR_LEN); + DIE(!array, "Error when allocating array"); + + generate_random_array(ARR_LEN, array); + + gettimeofday(&start, NULL); + + for (int i = 0; i < ARR_LEN; i++) + result += array[i]; + + gettimeofday(&stop, NULL); + + time = 1000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) / 1000; + + printf("Array sum is %ld\nTime spent: %lu miliseconds\n", result, time); + + free(array); + + return 0; +} diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_threads.c b/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_threads.c new file mode 100644 index 0000000000..e742e78964 --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_threads.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include + +#include "include/generate_random_array.h" +#include "include/array_utils.h" +#include "utils/utils.h" + +struct thread_args { + int *array; + size_t start; + size_t end; + long *results; + size_t tid; +}; + +static void *calculate_array_part_sum(void *varg) +{ + struct thread_args *arg = (struct thread_args *)varg; + long sum_array = 0; + + for (size_t i = arg->start; i < arg->end; i++) + sum_array += arg->array[i]; + + arg->results[arg->tid] = sum_array; + + return NULL; +} + +int main(int argc, char *argv[]) +{ + int num_threads; + struct timeval start, stop; + long time; + pthread_t *threads; + long *results; + struct thread_args *args; + int ret; + long final_result = 0; + int *array; + + if (argc < 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(1); + } + + num_threads = atoi(argv[1]); + + array = malloc(sizeof(*array) * ARR_LEN); + DIE(!array, "Error when allocating array"); + + threads = malloc(sizeof(*threads) * num_threads); + if (!threads) { + fprintf(stderr, "Error when allocating threads: %s", strerror(errno)); + ret = errno; + goto error_malloc_threads; + } + + results = malloc(sizeof(*results) * num_threads); + if (!results) { + fprintf(stderr, "Error when allocating results: %s", strerror(errno)); + ret = errno; + goto error_malloc_results; + } + + args = malloc(sizeof(*args) * num_threads); + if (!args) { + fprintf(stderr, "Error when allocating thread args: %s", strerror(errno)); + ret = errno; + goto error_malloc_args; + } + + generate_random_array(ARR_LEN, array); + + gettimeofday(&start, NULL); + + for (int i = 0; i < num_threads; i++) { + size_t elems_per_thread = ARR_LEN / num_threads; + size_t start = i * elems_per_thread; + size_t end = min((size_t)ARR_LEN, (i + 1) * elems_per_thread); + + args[i].array = array; + args[i].start = start; + args[i].end = end; + args[i].results = results; + args[i].tid = i; + + ret = pthread_create(&threads[i], NULL, calculate_array_part_sum, (void *)&args[i]); + if (ret != 0) { + fprintf(stderr, "Error when creating thread #%d: %s", i, strerror(errno)); + goto error_pthread_create; + } + } + + for (int i = 0; i < num_threads; i++) + pthread_join(threads[i], NULL); + + for (int i = 0; i < num_threads; i++) + final_result += results[i]; + + gettimeofday(&stop, NULL); + + time = 1000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) / 1000; + + printf("Array sum is %ld\nTime spent: %lu ms\n", final_result, time); + +error_pthread_create: + free(args); + +error_malloc_args: + free(results); + +error_malloc_results: + free(threads); + +error_malloc_threads: + free(array); + + return ret; +} diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_threads_openmp.c b/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_threads_openmp.c new file mode 100644 index 0000000000..aec15f30a7 --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/sum_array_threads_openmp.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include + +#include "include/generate_random_array.h" +#include "include/array_utils.h" +#include "utils/utils.h" + +int main(int argc, char *argv[]) +{ + struct timeval start, stop; + long time, result = 0; + int num_threads; + int *array; + + if (argc < 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(1); + } + + array = malloc(sizeof(*array) * ARR_LEN); + DIE(!array, "Error when allocating array"); + + generate_random_array(ARR_LEN, array); + + num_threads = atoi(argv[1]); + + omp_set_num_threads(num_threads); + + gettimeofday(&start, NULL); + +#pragma omp parallel for reduction(+ \ + : result) + for (int i = 0; i < ARR_LEN; i++) + result += array[i]; + + gettimeofday(&stop, NULL); + + time = 1000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) / 1000; + + printf("Array sum is %ld\nTime spent: %lu ms\n", result, time); + + free(array); + + return 0; +} diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/utils/log/CPPLINT.cfg b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/utils/log/log.c b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/utils/log/log.h b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.c b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.h b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/utils/utils.h b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/compute/lab/solution/wait-for-me/.gitignore b/chapters/compute/threads/guides/wait-for-me-threads/.gitignore similarity index 100% rename from content/chapters/compute/lab/solution/wait-for-me/.gitignore rename to chapters/compute/threads/guides/wait-for-me-threads/.gitignore diff --git a/chapters/compute/threads/guides/wait-for-me-threads/README.md b/chapters/compute/threads/guides/wait-for-me-threads/README.md new file mode 100644 index 0000000000..594ac57306 --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/README.md @@ -0,0 +1,20 @@ +# Wait for Me Once More + +Go to `support/wait-for-me/wait_for_me_threads.c`. +Spawn a thread that executes the `negate_array()` function. +For now, do not wait for it to finish; +simply start it. + +Compile the code and run the resulting executable several times. +See that the negative numbers appear from different indices. +This is precisely the nondeterminism that we talked about [in the previous section](tasks/wait-for-me.md). + +Now wait for that thread to finish and see that all the printed numbers are consistently negative. + +As you can see, waiting is a very coarse form of synchronization. +If we only use waiting, we can expect no speedup as a result of parallelism, because one thread must finish completely before another can continue. +We will discuss more fine-grained synchronization mechanisms [later in this lab](reading/synchronization.md). + +Also, at this point, you might be wondering why this exercise is written in D, while [the same exercise, but with processes](reading/processes.md#practice-wait-for-me) was written in Python. +There is a very good reason for this and has to do with how threads are synchronized by default in Python. +You can find out what this is about [in the Arena section](gil), after you have completed the [Synchronization section](reading/synchronization.md). diff --git a/content/chapters/compute/lab/support/wait-for-me/.gitignore b/chapters/compute/threads/guides/wait-for-me-threads/solution/.gitignore similarity index 100% rename from content/chapters/compute/lab/support/wait-for-me/.gitignore rename to chapters/compute/threads/guides/wait-for-me-threads/solution/.gitignore diff --git a/chapters/compute/threads/guides/wait-for-me-threads/solution/Makefile b/chapters/compute/threads/guides/wait-for-me-threads/solution/Makefile new file mode 120000 index 0000000000..a39be4f20d --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/solution/Makefile @@ -0,0 +1 @@ +../support/Makefile \ No newline at end of file diff --git a/chapters/compute/threads/guides/wait-for-me-threads/solution/wait_for_me_processes.py b/chapters/compute/threads/guides/wait-for-me-threads/solution/wait_for_me_processes.py new file mode 100644 index 0000000000..20fd47f22f --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/solution/wait_for_me_processes.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: BSD-3-Clause + +from multiprocessing import Process +from sys import argv +from time import sleep + + +def write_message_to_file(filename): + sleep(1) + open(filename, "w").write("Now you see me, now you don't\n") + + +def main(): + if len(argv) < 2: + print(f"Usage: {argv[0]} ") + return 1 + + file_name = argv[1] + child = Process(target=write_message_to_file, args=(file_name,)) + child.start() + + # TODO: Fix the `FileNotFoundError` raised below. + child.join() + + file_content = open(file_name).read() + print(f"File content is: '{file_content}'") + + +if __name__ == "__main__": + exit(main()) diff --git a/content/chapters/compute/lab/solution/wait-for-me/wait_for_me_threads.d b/chapters/compute/threads/guides/wait-for-me-threads/solution/wait_for_me_threads.d similarity index 100% rename from content/chapters/compute/lab/solution/wait-for-me/wait_for_me_threads.d rename to chapters/compute/threads/guides/wait-for-me-threads/solution/wait_for_me_threads.d diff --git a/chapters/compute/threads/guides/wait-for-me-threads/support/Makefile b/chapters/compute/threads/guides/wait-for-me-threads/support/Makefile new file mode 100644 index 0000000000..ae238bca75 --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/support/Makefile @@ -0,0 +1,38 @@ +BINARY = wait_for_me_threads + +all: $(BINARY) + +CC = gcc + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR).. +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +SRCS = $(wildcard *.c) +OBJS = $(SRCS:.c=.o) + +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + @make -C $(LOGGER_DIR) $(LOGGER_OBJ) + +$(OBJS): %.o: %.c + +clean:: + -rm -f $(OBJS) $(LOGGER) + +.PHONY: clean + +$(BINARY): $(OBJS) $(LOGGER) + $(CC) $^ $(LDFLAGS) -o $@ $(LDLIBS) + +clean:: + -rm -f $(BINARY) + +.PHONY: all clean diff --git a/chapters/compute/threads/guides/wait-for-me-threads/support/utils/log/CPPLINT.cfg b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/compute/threads/guides/wait-for-me-threads/support/utils/log/log.c b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/compute/threads/guides/wait-for-me-threads/support/utils/log/log.h b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.c b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.h b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/compute/threads/guides/wait-for-me-threads/support/utils/utils.h b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/compute/threads/guides/wait-for-me-threads/support/wait_for_me_processes.py b/chapters/compute/threads/guides/wait-for-me-threads/support/wait_for_me_processes.py new file mode 100644 index 0000000000..d3b505b11f --- /dev/null +++ b/chapters/compute/threads/guides/wait-for-me-threads/support/wait_for_me_processes.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: BSD-3-Clause + +from multiprocessing import Process +from sys import argv +from time import sleep + + +def write_message_to_file(filename): + sleep(1) + open(filename, "w").write("Now you see me, now you don't") + + +def main(): + if len(argv) < 2: + print(f"Usage: {argv[0]} ") + return 1 + + file_name = argv[1] + child = Process(target=write_message_to_file, args=(file_name,)) + child.start() + + # TODO: Fix the `FileNotFoundError` raised below. + + file_content = open(file_name).readline() + print(f"File content is: '{file_content}'") + + +if __name__ == "__main__": + exit(main()) diff --git a/content/chapters/compute/lab/support/wait-for-me/wait_for_me_threads.c b/chapters/compute/threads/guides/wait-for-me-threads/support/wait_for_me_threads.c similarity index 100% rename from content/chapters/compute/lab/support/wait-for-me/wait_for_me_threads.c rename to chapters/compute/threads/guides/wait-for-me-threads/support/wait_for_me_threads.c diff --git a/content/chapters/compute/lecture/media/app-process-thread.svg b/chapters/compute/threads/media/app-process-thread.svg similarity index 100% rename from content/chapters/compute/lecture/media/app-process-thread.svg rename to chapters/compute/threads/media/app-process-thread.svg diff --git a/content/chapters/compute/lecture/media/thread-states.svg b/chapters/compute/threads/media/thread-states.svg similarity index 100% rename from content/chapters/compute/lecture/media/thread-states.svg rename to chapters/compute/threads/media/thread-states.svg diff --git a/chapters/compute/threads/reading/threads.md b/chapters/compute/threads/reading/threads.md new file mode 100644 index 0000000000..4b190f2244 --- /dev/null +++ b/chapters/compute/threads/reading/threads.md @@ -0,0 +1,60 @@ +# Threads + +## Threads vs Processes + +So why use the implementation that spawns more processes if it's slower than the one using threads? +The table below lists the differences between threads and processes. +Generally, if we only want to do some computing, we use threads. +If we need to drastically change the behaviour of the program, we need a new program altogether, or we need more than computing (e.g. communication on the network to create a computing cluster), we use processes. + +| PROCESS | THREAD | +| :--------------------------------------------------- | :---------------------------------------------------------------------- | +| independent | part of a process | +| collection of threads | shares VAS with other threads | +| slower creation (new page table must be created) | faster creation | +| longer context switch duration (TLB must be flushed) | shorter context switch duration (part of the same process, so same TLB) | +| ending means ending all threads | other threads continue when finished | + +### Safety + +Compile and run the two programs in `sum-array-bugs/support/seg-fault/`, first with 2 processes and threads and then with 4. +They do the same thing as before: compute the sum of the elements in an array, but with a twist: each of them contains a bug causing a segfault. +Notice that `sum_array_threads` doesn't print anything with 4 threads, but merely a "Segmentation fault" message. +On the other hand, `sum_array_processes` prints a sum and a running time, albeit different from the sums we've seen so far. + +The reason is that signals such as `SIGSEGV`, which is used when a segmentation fault happens affect the entire process that handles them. +Therefore, when we split our workload between several threads and one of them causes an error such as a segfault, that error is going to terminate the entire process. +The same thing happens when we use processes instead of threads: one process causes an error, which gets it killed, but the other processes continue their work unhindered. +This is why we end up with a lower sum in the end: because one process died too early and didn't manage to write the partial sum it had computed to the `results` array. + +## Memory Layout of Multithreaded Programs + +When a new thread is created, a new stack is allocated for a thread. +The default stack size if `8 MB` / `8192 KB`: + +```console +student@os:~$ ulimit -s +8192 +``` + +Enter the `multithreaded/support/` directory to observe the update of the memory layout when creating new threads. + +Build the `multithreaded` executable: + +```console +student@os:~/.../multithreaded/support$ make +``` + +Start the program: + +```console +student@os:~/.../multithreaded/support$ ./multithreaded +Press key to start creating threads ... +[...] +``` + +And investigate it with `pmap` on another console, while pressing a key to create new threads. + +As you can see, there is a new `8192 KB` area created for every thread, also increasing the total virtual size. + +[Quiz](../drills/questions/thread-memory.md) diff --git a/content/chapters/compute/lecture/slides/threads.md b/chapters/compute/threads/slides/threads.md similarity index 91% rename from content/chapters/compute/lecture/slides/threads.md rename to chapters/compute/threads/slides/threads.md index 25d7898e1c..4dabe04958 100644 --- a/content/chapters/compute/lecture/slides/threads.md +++ b/chapters/compute/threads/slides/threads.md @@ -25,9 +25,9 @@ ### App, Process, Thread - The Bigger Picture -* [Quiz](../quiz/threads-shared-data.md) +* [Quiz](../drills/questions/threads-shared-data.md) -![Application Architecture](media/app-process-thread.svg) +![Application Architecture](../media/app-process-thread.svg) --- diff --git a/content/chapters/compute/lab/quiz/fiber-strace.md b/chapters/compute/user-level-threads/drills/questions/fiber-strace.md similarity index 100% rename from content/chapters/compute/lab/quiz/fiber-strace.md rename to chapters/compute/user-level-threads/drills/questions/fiber-strace.md diff --git a/content/chapters/compute/lab/quiz/sleeping-on-a-fiber.md b/chapters/compute/user-level-threads/drills/questions/sleeping-on-a-fiber.md similarity index 100% rename from content/chapters/compute/lab/quiz/sleeping-on-a-fiber.md rename to chapters/compute/user-level-threads/drills/questions/sleeping-on-a-fiber.md diff --git a/chapters/compute/user-level-threads/guides/user-level-threads/README.md b/chapters/compute/user-level-threads/guides/user-level-threads/README.md new file mode 100644 index 0000000000..e73ae7dd80 --- /dev/null +++ b/chapters/compute/user-level-threads/guides/user-level-threads/README.md @@ -0,0 +1,28 @@ +# Interaction Between Threads and Fibers + +As we mentioned before, multiple fibers can run on the same thread, and a scheduler is implemented on each thread. +By default, the scheduling algorithm is [`round_robin`](https://www.guru99.com/round-robin-scheduling-example.html). +It runs the fibers, in the order of their creation, until they yield or finish their work. +If a fiber yields, it is placed at the back of the round-robin queue. +Using this scheduler, each thread only uses its fibers; +if one thread has more work to do than another, bad luck. +This may lead to starvation. + +But there are other scheduler implementations, such as `shared_work` and `work_stealing`. +Follow the `user-level-threads/support/threads_and_fibers.cc` implementation. +It creates multiple fibers and threads, and uses the `shared_work` scheduler to balance the workload between the threads. +Each main fiber, from each thread, is suspended until all worker fibers have completed their work, using a condition variable. + +```cpp +cnd_count.wait( lk, []{ return 0 == fiber_count; } ); +``` + +The program also uses `thread local storage` and `fiber local storage` to store the ID of each thread / fiber. + +Now change the `shared_work` scheduler into the `work_stealing` one. +It takes a parameter, the number of threads that will use that scheduler. + +Compile, rerun and note the differences. +The `work_stealing` scheduler, as the name suggests, will "steal" fibers from other schedulers. +So, if the `shared_work` scheduler tried to balance the available work between the available threads, the `work_stealing` one will focus on having as many threads as possible on 100% workload. +Vary the number of threads and fibers, and the workload (maybe put each fibre to do some computational-intensive work), and observe the results. diff --git a/content/chapters/compute/lab/support/user-level-threads/.gitignore b/chapters/compute/user-level-threads/guides/user-level-threads/support/.gitignore similarity index 100% rename from content/chapters/compute/lab/support/user-level-threads/.gitignore rename to chapters/compute/user-level-threads/guides/user-level-threads/support/.gitignore diff --git a/content/chapters/compute/lab/support/user-level-threads/CMakeLists.txt b/chapters/compute/user-level-threads/guides/user-level-threads/support/CMakeLists.txt similarity index 100% rename from content/chapters/compute/lab/support/user-level-threads/CMakeLists.txt rename to chapters/compute/user-level-threads/guides/user-level-threads/support/CMakeLists.txt diff --git a/content/chapters/compute/lab/support/user-level-threads/CPPLINT.cfg b/chapters/compute/user-level-threads/guides/user-level-threads/support/CPPLINT.cfg similarity index 100% rename from content/chapters/compute/lab/support/user-level-threads/CPPLINT.cfg rename to chapters/compute/user-level-threads/guides/user-level-threads/support/CPPLINT.cfg diff --git a/content/chapters/compute/lab/support/user-level-threads/simple.cc b/chapters/compute/user-level-threads/guides/user-level-threads/support/simple.cc similarity index 100% rename from content/chapters/compute/lab/support/user-level-threads/simple.cc rename to chapters/compute/user-level-threads/guides/user-level-threads/support/simple.cc diff --git a/content/chapters/compute/lab/support/user-level-threads/sum.cc b/chapters/compute/user-level-threads/guides/user-level-threads/support/sum.cc similarity index 100% rename from content/chapters/compute/lab/support/user-level-threads/sum.cc rename to chapters/compute/user-level-threads/guides/user-level-threads/support/sum.cc diff --git a/content/chapters/compute/lab/support/user-level-threads/threads_and_fibers.cc b/chapters/compute/user-level-threads/guides/user-level-threads/support/threads_and_fibers.cc similarity index 100% rename from content/chapters/compute/lab/support/user-level-threads/threads_and_fibers.cc rename to chapters/compute/user-level-threads/guides/user-level-threads/support/threads_and_fibers.cc diff --git a/content/chapters/compute/lab/support/user-level-threads/yield_barrier.cc b/chapters/compute/user-level-threads/guides/user-level-threads/support/yield_barrier.cc similarity index 100% rename from content/chapters/compute/lab/support/user-level-threads/yield_barrier.cc rename to chapters/compute/user-level-threads/guides/user-level-threads/support/yield_barrier.cc diff --git a/content/chapters/compute/lab/support/user-level-threads/yield_launch.cc b/chapters/compute/user-level-threads/guides/user-level-threads/support/yield_launch.cc similarity index 100% rename from content/chapters/compute/lab/support/user-level-threads/yield_launch.cc rename to chapters/compute/user-level-threads/guides/user-level-threads/support/yield_launch.cc diff --git a/chapters/compute/user-level-threads/reading/user-level-threads.md b/chapters/compute/user-level-threads/reading/user-level-threads.md new file mode 100644 index 0000000000..b663825f8f --- /dev/null +++ b/chapters/compute/user-level-threads/reading/user-level-threads.md @@ -0,0 +1,78 @@ +# User-Level Threads + +User-level threads differ from the threads you are used to (kernel-level threads, those created by `pthread_create`). +This kind of threads are scheduled by an user-level scheduler, and can run on the same kernel-level thread. +From now on, we will refer to user-level threads as fibers, and kernel-level threads as simply threads. + +We will use the fiber implementation from `libboost`. +This implementation uses a cooperative scheduler on each thread, meaning that each fiber has to yield, in order for other fiber to be executed. +We will also use C++, and the standard `thread` implementation. + +## Prerequisites + +Unless you are using the OS docker image, you will need to install `cmake` and `libboost`. +You can do this with the following command: + +```console +student@os:~$ sudo apt-get install cmake libboost-context-dev libboost-fiber-dev +``` + +## Creation + +Follow the `user-level-threads/support/simple.cc` implementation. +It creates `NUM_FIBERS` fibers, that each prints "Hello World". +To compile and run the program, do the following steps: + +```console +student@os:~/.../user-level-threads/support$ mkdir build/ +student@os:~/.../user-level-threads/support$ cd build/ +student@os:~/.../user-level-threads/support$ cmake -S .. -B . +student@os:~/.../user-level-threads/support$ make +student@os:~/.../user-level-threads/support$ ./simple +``` + +The `cmake` step must be executed only once. +After modifying the source files, it is enough to run `make`. + +### Practice: Sleeper Fiber + +Add in `user-level-threads/support/simple.cc` a fiber that sleeps for 5 seconds, before the other ones are created. +What happens? +Answer in this [quiz](../drills/questions/sleeping-on-a-fiber.md). + +## No system calls + +Use `strace` to find calls to `clone()` in the execution of `simple`. +Can you find any? +Provide your answer in this [quiz](../drills/questions/fiber-strace.md) +Remember that `clone()` is the system call used to create **kernel-level** threads, as pointed out [here](clone). + +## Synchronization + +By default, the fibers that run on the same thread are synchronized - no race-conditions can occur. +This is illustrated by the `user-level-threads/support/sum.cc` implementation. + +The user can, however, implement further synchronization, by using the `yield()` call, or classic synchronization methods, like mutexes, barriers and condition variables. + +### Yielding + +As the scheduler is cooperative, each fiber can yield (or not), to allow another fiber to run. +Follow the `user-level-threads/support/yield_launch.cc` implementation and run it. +Note the `boost::fibers::launch::dispatch` parameter provided to the fiber constructor. +It notifies the scheduler to start the fibre as soon as it is created. +In order to explain the output, we must consider that the fibers are created by a **main fiber**, that is scheduled along with the others, in this case. + +#### Practice + +Modify the launch parameter into `boost::fibers::launch::post`, compile and notice the differences. +The `post` parameter notifies the scheduler not to start the fibers immediately, but rather place them into an execution queue. +Their execution will start after the main fiber calls the `join()` function. + +### Barriers + +Follow the `user-level-threads/support/yield_barrier.cc` implementation. +It uses a barrier to achieve the same result as the previous implementation, that used `post` as the launch parameter. + +### C++ unique_lock + +`unique_lock` is a type of mutex that is unlocked automatically when the end of its scope is reached (end of function or bracket-pair). diff --git a/chapters/data/memory-security/drills/tasks/exec-shellcode/solution/utils/log/log.h b/chapters/data/memory-security/drills/tasks/exec-shellcode/solution/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/memory-security/drills/tasks/exec-shellcode/solution/utils/log/log.h +++ b/chapters/data/memory-security/drills/tasks/exec-shellcode/solution/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/memory-security/drills/tasks/exec-shellcode/support/utils/log/log.h b/chapters/data/memory-security/drills/tasks/exec-shellcode/support/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/memory-security/drills/tasks/exec-shellcode/support/utils/log/log.h +++ b/chapters/data/memory-security/drills/tasks/exec-shellcode/support/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/alloc_size.c b/chapters/data/process-memory/drills/tasks/alloc-size/support/alloc_size.c index 41ee007288..f675ff1042 100644 --- a/chapters/data/process-memory/drills/tasks/alloc-size/support/alloc_size.c +++ b/chapters/data/process-memory/drills/tasks/alloc-size/support/alloc_size.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/log.h +++ b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/process-memory/drills/tasks/copy/solution/src/mmap_copy.c b/chapters/data/process-memory/drills/tasks/copy/solution/src/mmap_copy.c index 333f68bf54..506c1512e7 100644 --- a/chapters/data/process-memory/drills/tasks/copy/solution/src/mmap_copy.c +++ b/chapters/data/process-memory/drills/tasks/copy/solution/src/mmap_copy.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/data/process-memory/drills/tasks/copy/solution/src/read_write_copy.c b/chapters/data/process-memory/drills/tasks/copy/solution/src/read_write_copy.c index d9b82d7af4..b372ac3a61 100644 --- a/chapters/data/process-memory/drills/tasks/copy/solution/src/read_write_copy.c +++ b/chapters/data/process-memory/drills/tasks/copy/solution/src/read_write_copy.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/data/process-memory/drills/tasks/copy/solution/src/utils/log/log.h b/chapters/data/process-memory/drills/tasks/copy/solution/src/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/process-memory/drills/tasks/copy/solution/src/utils/log/log.h +++ b/chapters/data/process-memory/drills/tasks/copy/solution/src/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/hello.c b/chapters/data/process-memory/drills/tasks/memory-areas/support/hello.c index c19721ba7d..f413109721 100644 --- a/chapters/data/process-memory/drills/tasks/memory-areas/support/hello.c +++ b/chapters/data/process-memory/drills/tasks/memory-areas/support/hello.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/log.h +++ b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/hello.c b/chapters/data/process-memory/drills/tasks/modify-areas/support/hello.c index f97da2ba7c..fed32f9d5b 100644 --- a/chapters/data/process-memory/drills/tasks/modify-areas/support/hello.c +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/hello.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/log.h +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/process-memory/drills/tasks/page-mapper/solution/utils/log/log.h b/chapters/data/process-memory/drills/tasks/page-mapper/solution/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/process-memory/drills/tasks/page-mapper/solution/utils/log/log.h +++ b/chapters/data/process-memory/drills/tasks/page-mapper/solution/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/process-memory/drills/tasks/page-mapper/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/page-mapper/support/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/process-memory/drills/tasks/page-mapper/support/utils/log/log.h +++ b/chapters/data/process-memory/drills/tasks/page-mapper/support/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/process-memory/drills/tasks/reference-counting/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/reference-counting/support/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/process-memory/drills/tasks/reference-counting/support/utils/log/log.h +++ b/chapters/data/process-memory/drills/tasks/reference-counting/support/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/hello.c b/chapters/data/process-memory/drills/tasks/static-dynamic/support/hello.c index c19721ba7d..f413109721 100644 --- a/chapters/data/process-memory/drills/tasks/static-dynamic/support/hello.c +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/support/hello.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/log.h +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/working-with-memory/drills/tasks/access-counter/solution/utils/log/log.h b/chapters/data/working-with-memory/drills/tasks/access-counter/solution/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/working-with-memory/drills/tasks/access-counter/solution/utils/log/log.h +++ b/chapters/data/working-with-memory/drills/tasks/access-counter/solution/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/working-with-memory/drills/tasks/access-counter/support/utils/log/log.h b/chapters/data/working-with-memory/drills/tasks/access-counter/support/utils/log/log.h index c8d1dee06a..7acb55aab6 100644 --- a/chapters/data/working-with-memory/drills/tasks/access-counter/support/utils/log/log.h +++ b/chapters/data/working-with-memory/drills/tasks/access-counter/support/utils/log/log.h @@ -24,27 +24,27 @@ extern "C" #define LOG_VERSION "0.1.0" - typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; - } log_Event; +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; - typedef void (*log_LogFn)(log_Event *ev); - typedef void (*log_LockFn)(bool lock, void *udata); +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); - enum { - LOG_TRACE, - LOG_DEBUG, - LOG_INFO, - LOG_WARN, - LOG_ERROR, - LOG_FATAL - }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) @@ -53,14 +53,14 @@ extern "C" #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) - const char *log_level_string(int level); - void log_set_lock(log_LockFn fn, void *udata); - void log_set_level(int level); - void log_set_quiet(bool enable); - int log_add_callback(log_LogFn fn, void *udata, int level); - int log_add_fp(FILE *fp, int level); +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); - void log_log(int level, const char *file, int line, const char *fmt, ...); +void log_log(int level, const char *file, int line, const char *fmt, ...); #ifdef __cplusplus } diff --git a/chapters/data/working-with-memory/drills/tasks/memory-access/solution/src/mem_access.c b/chapters/data/working-with-memory/drills/tasks/memory-access/solution/src/mem_access.c index 47229acfa8..e53554f3b8 100644 --- a/chapters/data/working-with-memory/drills/tasks/memory-access/solution/src/mem_access.c +++ b/chapters/data/working-with-memory/drills/tasks/memory-access/solution/src/mem_access.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include diff --git a/chapters/data/working-with-memory/drills/tasks/memory-protection/solution/src/mem_prot.c b/chapters/data/working-with-memory/drills/tasks/memory-protection/solution/src/mem_prot.c index 292d5a3a24..56ef2ffb62 100644 --- a/chapters/data/working-with-memory/drills/tasks/memory-protection/solution/src/mem_prot.c +++ b/chapters/data/working-with-memory/drills/tasks/memory-protection/solution/src/mem_prot.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include diff --git a/chapters/data/working-with-memory/guides/memory-alloc/support/c_memory_alloc.c b/chapters/data/working-with-memory/guides/memory-alloc/support/c_memory_alloc.c index b6cc470a14..c2b27ebf81 100644 --- a/chapters/data/working-with-memory/guides/memory-alloc/support/c_memory_alloc.c +++ b/chapters/data/working-with-memory/guides/memory-alloc/support/c_memory_alloc.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/data/working-with-memory/guides/memory-vuln/support/c_memory_vuln.c b/chapters/data/working-with-memory/guides/memory-vuln/support/c_memory_vuln.c index 4b30e79035..4bf4c36f91 100644 --- a/chapters/data/working-with-memory/guides/memory-vuln/support/c_memory_vuln.c +++ b/chapters/data/working-with-memory/guides/memory-vuln/support/c_memory_vuln.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/software-stack/libc/drills/tasks/common-functions/solution/src/syscall.s b/chapters/software-stack/libc/drills/tasks/common-functions/solution/src/syscall.s index 93647c320e..1e1c6ab84a 100644 --- a/chapters/software-stack/libc/drills/tasks/common-functions/solution/src/syscall.s +++ b/chapters/software-stack/libc/drills/tasks/common-functions/solution/src/syscall.s @@ -1,4 +1,4 @@ -; SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause */ section .text diff --git a/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/log/log.c b/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/log/log.c index 3aebbfc1a4..ac65f4ed33 100644 --- a/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/log/log.c +++ b/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/log/log.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: BSD-3-Clause + /* * Copyright (c) 2020 rxi * @@ -27,144 +29,169 @@ #define MAX_CALLBACKS 32 typedef struct { - log_LogFn fn; - void *udata; - int level; + log_LogFn fn; + void *udata; + int level; } Callback; -static struct { - void *udata; - log_LockFn lock; - int level; - bool quiet; - Callback callbacks[MAX_CALLBACKS]; +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; } L; - -static const char *level_strings[] = { - "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" -}; +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; #ifdef LOG_USE_COLOR -static const char *level_colors[] = { - "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" -}; +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; #endif +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; -static void stdout_callback(log_Event *ev) { - char buf[16]; - buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; #ifdef LOG_USE_COLOR - fprintf( - ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", - buf, level_colors[ev->level], level_strings[ev->level], - ev->file, ev->line); + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); #else - fprintf( - ev->udata, "%s %-5s %s:%d: ", - buf, level_strings[ev->level], ev->file, ev->line); + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); #endif - vfprintf(ev->udata, ev->fmt, ev->ap); - fprintf(ev->udata, "\n"); - fflush(ev->udata); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); } - -static void file_callback(log_Event *ev) { - char buf[64]; - buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; - fprintf( - ev->udata, "%s %-5s %s:%d: ", - buf, level_strings[ev->level], ev->file, ev->line); - vfprintf(ev->udata, ev->fmt, ev->ap); - fprintf(ev->udata, "\n"); - fflush(ev->udata); +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); } - -static void lock(void) { - if (L.lock) { L.lock(true, L.udata); } +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); } - -static void unlock(void) { - if (L.lock) { L.lock(false, L.udata); } +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); } - -const char* log_level_string(int level) { - return level_strings[level]; +const char* +log_level_string(int level) +{ + return level_strings[level]; } - -void log_set_lock(log_LockFn fn, void *udata) { - L.lock = fn; - L.udata = udata; +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; } - -void log_set_level(int level) { - L.level = level; +void +log_set_level(int level) +{ + L.level = level; } - -void log_set_quiet(bool enable) { - L.quiet = enable; +void +log_set_quiet(bool enable) +{ + L.quiet = enable; } - -int log_add_callback(log_LogFn fn, void *udata, int level) { - for (int i = 0; i < MAX_CALLBACKS; i++) { - if (!L.callbacks[i].fn) { - L.callbacks[i] = (Callback) { fn, udata, level }; - return 0; - } - } - return -1; +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; } - -int log_add_fp(FILE *fp, int level) { - return log_add_callback(file_callback, fp, level); +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); } +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); -static void init_event(log_Event *ev, void *udata) { - if (!ev->time) { - time_t t = time(NULL); - ev->time = localtime(&t); - } - ev->udata = udata; + ev->time = localtime(&t); + } + ev->udata = udata; } - -void log_log(int level, const char *file, int line, const char *fmt, ...) { - log_Event ev = { - .fmt = fmt, - .file = file, - .line = line, - .level = level, - }; - - lock(); - - if (!L.quiet && level >= L.level) { - init_event(&ev, stderr); - va_start(ev.ap, fmt); - stdout_callback(&ev); - va_end(ev.ap); - } - - for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { - Callback *cb = &L.callbacks[i]; - if (level >= cb->level) { - init_event(&ev, cb->udata); - va_start(ev.ap, fmt); - cb->fn(&ev); - va_end(ev.ap); - } - } - - unlock(); +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); } diff --git a/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/log/log.h b/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/log/log.h index 1229b481dd..7acb55aab6 100644 --- a/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/log/log.h +++ b/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/log/log.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + /** * Copyright (c) 2020 rxi * @@ -10,40 +12,48 @@ #ifndef LOG_H #define LOG_H -#include #include #include +#include #include #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif #define LOG_VERSION "0.1.0" typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int line; - int level; + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; } log_Event; typedef void (*log_LogFn)(log_Event *ev); typedef void (*log_LockFn)(bool lock, void *udata); -enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) -#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) -#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) -const char* log_level_string(int level); +const char *log_level_string(int level); void log_set_lock(log_LockFn fn, void *udata); void log_set_level(int level); void log_set_quiet(bool enable); diff --git a/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.c b/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.h b/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/software-stack/libc/drills/tasks/libc/support/hello.c b/chapters/software-stack/libc/drills/tasks/libc/support/hello.c index 356134afc8..960fc0d963 100644 --- a/chapters/software-stack/libc/drills/tasks/libc/support/hello.c +++ b/chapters/software-stack/libc/drills/tasks/libc/support/hello.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include diff --git a/chapters/software-stack/libc/drills/tasks/libc/support/main_printf.c b/chapters/software-stack/libc/drills/tasks/libc/support/main_printf.c index eb0f4539e5..d88f1e52d0 100644 --- a/chapters/software-stack/libc/drills/tasks/libc/support/main_printf.c +++ b/chapters/software-stack/libc/drills/tasks/libc/support/main_printf.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include "string.h" diff --git a/chapters/software-stack/libc/drills/tasks/libc/support/main_string.c b/chapters/software-stack/libc/drills/tasks/libc/support/main_string.c index 54fd67c81c..9113b0a190 100644 --- a/chapters/software-stack/libc/drills/tasks/libc/support/main_string.c +++ b/chapters/software-stack/libc/drills/tasks/libc/support/main_string.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/software-stack/libc/drills/tasks/libc/support/memory.c b/chapters/software-stack/libc/drills/tasks/libc/support/memory.c index 40d877c2cb..4758e3163d 100644 --- a/chapters/software-stack/libc/drills/tasks/libc/support/memory.c +++ b/chapters/software-stack/libc/drills/tasks/libc/support/memory.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include diff --git a/chapters/software-stack/libc/drills/tasks/libc/support/vendetta.c b/chapters/software-stack/libc/drills/tasks/libc/support/vendetta.c index b76a3c86b6..65cd97de52 100644 --- a/chapters/software-stack/libc/drills/tasks/libc/support/vendetta.c +++ b/chapters/software-stack/libc/drills/tasks/libc/support/vendetta.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/software-stack/libraries/guides/static-dynamic/support/hello.c b/chapters/software-stack/libraries/guides/static-dynamic/support/hello.c index 6b2937d13f..772630f360 100644 --- a/chapters/software-stack/libraries/guides/static-dynamic/support/hello.c +++ b/chapters/software-stack/libraries/guides/static-dynamic/support/hello.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include diff --git a/chapters/software-stack/system-calls/drills/tasks/basic-syscall/solution/src/hello.asm b/chapters/software-stack/system-calls/drills/tasks/basic-syscall/solution/src/hello.asm index dc45647f1a..0acee2548d 100644 --- a/chapters/software-stack/system-calls/drills/tasks/basic-syscall/solution/src/hello.asm +++ b/chapters/software-stack/system-calls/drills/tasks/basic-syscall/solution/src/hello.asm @@ -1,4 +1,4 @@ -; SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause */ section .bss diff --git a/chapters/software-stack/system-calls/drills/tasks/libcall-syscall/support/call.c b/chapters/software-stack/system-calls/drills/tasks/libcall-syscall/support/call.c index 47387b2249..81cade84a6 100644 --- a/chapters/software-stack/system-calls/drills/tasks/libcall-syscall/support/call.c +++ b/chapters/software-stack/system-calls/drills/tasks/libcall-syscall/support/call.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/software-stack/system-calls/drills/tasks/libcall-syscall/support/call2.c b/chapters/software-stack/system-calls/drills/tasks/libcall-syscall/support/call2.c index 84f4204e26..93e4a56432 100644 --- a/chapters/software-stack/system-calls/drills/tasks/libcall-syscall/support/call2.c +++ b/chapters/software-stack/system-calls/drills/tasks/libcall-syscall/support/call2.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/software-stack/system-calls/drills/tasks/syscall-wrapper/solution/src/main.c b/chapters/software-stack/system-calls/drills/tasks/syscall-wrapper/solution/src/main.c index d599c36a15..fb2aaa95c4 100644 --- a/chapters/software-stack/system-calls/drills/tasks/syscall-wrapper/solution/src/main.c +++ b/chapters/software-stack/system-calls/drills/tasks/syscall-wrapper/solution/src/main.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/software-stack/system-calls/drills/tasks/syscall-wrapper/solution/src/syscall.asm b/chapters/software-stack/system-calls/drills/tasks/syscall-wrapper/solution/src/syscall.asm index a8a53cb9d7..d697eb7f77 100644 --- a/chapters/software-stack/system-calls/drills/tasks/syscall-wrapper/solution/src/syscall.asm +++ b/chapters/software-stack/system-calls/drills/tasks/syscall-wrapper/solution/src/syscall.asm @@ -1,4 +1,4 @@ -; SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause */ section .text diff --git a/config.yaml b/config.yaml index ff30c0d938..12950648f8 100644 --- a/config.yaml +++ b/config.yaml @@ -73,6 +73,47 @@ lab_structure: - memory-security.md - guides/buffer-overflow-leak.md - guides/buffer-overflow-overwrite.md + - title: Lab 6 - Multiprocess and Multithread + filename: lab6.md + content: + - tasks/sleepy.md + - tasks/wait-for-me-processes.md + - tasks/create-process.md + - tasks/mini-shell.md + - tasks/multithreaded.md + - tasks/sum-array.md + - tasks/sum-array-bugs.md + - hardware-perspective.md + - processes.md + - threads.md + - guides/create-process.md + - guides/sum-array-processes.md + - guides/system-dissected.md + - guides/sum-array-threads.md + - guides/wait-for-me-threads.md + - guides/clone.md + - title: Lab 7 - Copy-on-Write + filename: lab7.md + content: + - tasks/apache2.md + - tasks/page-faults.md + - tasks/shared-memory.md + - processes-threads-apache2.md + - copy-on-write.md + - guides/apache2.md + - guides/fork-faults.md + - title: Lab 8 - Syncronization + filename: lab8.md + content: + - tasks/race-condition.md + - tasks/threadsafe-data-struct.md + - tasks/libult.md + - synchronization.md + - user-level-threads.md + - scheduling.md + - guides/user-level-threads.md + - guides/libult.md + make_assets: plugin: command @@ -81,7 +122,7 @@ make_assets: locations: - chapters/software-stack - chapters/data - - content/chapters/compute/lecture + - chapters/compute - content/chapters/io/lecture - content/chapters/app-interact/lecture args: @@ -172,28 +213,56 @@ docusaurus: - Lab 3 - Memory: lab3.md - Lab 4 - Investigate Memory: lab4.md - Lab 5 - Memory Security: lab5.md + - Compute: + path: /build/prepare_view/.view + extra: + - media + subsections: + - Slides: /build/embed_reveal/Compute/Compute.mdx + - Overview: /build/prepare_view/.view/compute-overview.md + - Questions: + path: questions + subsections: + - Parent Faults Before Fork: parent-faults-before-fork.md + - Mmap Cow Flag: mmap-cow-flag.md + - Sections Always Shared: sections-always-shared.md + - Child Faults After Write: child-faults-after-write.md + - Seg Fault Exit Code: seg-fault-exit-code.md + - Threads Shared Data: threads-shared-data.md + - Thread Memory: thread-memory.md + - Apache2 Strace: apache2-strace.md + - Mini Shell Stops After Command: mini-shell-stops-after-command.md + - Cause of File Not Found Error: cause-of-file-not-found-error.md + - Process Creation: process-creation.md + - Who Calls Execve Parent: who-calls-execve-parent.md + - Exec Without Fork: exec-without-fork.md + - Create Sleepy Process Ending: create-sleepy-process-ending.md + - Processes Speedup: processes-speedup.md + - Parent of Sleep Processes: parent-of-sleep-processes.md + - Sleeping on a Fiber: sleeping-on-a-fiber.md + - Fiber Strace: fiber-strace.md + - Type of Scheduler in Libult: type-of-scheduler-in-libult.md + - Number of Running Ults: number-of-running-ults.md + - Why Use Completed Queue: why-use-completed-queue.md + - Ult Thread IDs: ult-thread-ids.md + - TCB Libult Unikraft: tcb-libult-unikraft.md + - Time Slice Value: time-slice-value.md + - Number of Running Threads: number-of-running-threads.md + - Notify Only With Mutex: notify-only-with-mutex.md + - TLS Var Copies: tls-var-copies.md + - TLS Synchronization: tls-synchronization.md + - Semaphore Equivalent: semaphore-equivalent.md + - Coarse vs Granular Critical Section: coarse-vs-granular-critical-section.md + - Not Race Condition: not-race-condition.md + - Lab 6 - Multiprocess and Multithread: lab6.md + - Lab 7 - Copy-on-Write: lab7.md + - Lab 8 - Syncronization: lab8.md - Lecture: path: /build/embed_reveal subsections: - - Compute: Compute/Compute.mdx - IO: IO/IO.mdx - Application Interaction: Application-Interaction/Application-Interaction.mdx # - Lab: - # - Compute: - # path: content/chapters/compute/lab/content - # extra: - # - ../media - # - ../quiz - # subsections: - # - Overview: overview.md - # - Hardware Perspective: hardware-perspective.md - # - Processes: processes.md - # - Threads: threads.md - # - Processes-threads-apache2: processes-threads-apache2.md - # - Copy-on-Write: copy-on-write.md - # - Synchronization: synchronization.md - # - User-Level Threads: user-level-threads.md - # - Arena: arena.md # - IO: # path: content/chapters/io/lab/content # extra: @@ -215,7 +284,6 @@ docusaurus: # - Zero-Copy: zero-copy.md # - Asynchronous IO: async-io.md # - IO Multiplexing: io-multiplexing.md - # - Arena: arena.md # - Application Interaction: # path: content/chapters/app-interact/lab/content # extra: @@ -228,7 +296,6 @@ docusaurus: # - The X Window System: x-window-system.md # - D-Bus: dbus.md # - OS Cloud: os-cloud.md - # - Arena: arena.md - Assignments: # These all have the same filename so we need to create separate # subdirectories for them, otherwise they will overwrite each other. @@ -279,7 +346,7 @@ docusaurus: - Rules and Grading: misc/rules-and-grading.md - Resources: misc/resources.md static_assets: - - slides/Compute: /build/make_assets/content/chapters/compute/lecture/_site + - slides/Compute: /build/make_assets/chapters/compute/_site - slides/Data: /build/make_assets/chapters/data/_site - slides/Software-Stack: /build/make_assets/chapters/software-stack/_site - slides/IO: /build/make_assets/content/chapters/io/lecture/_site diff --git a/content/chapters/app-interact/lab/solution/password-cracker/password-cracker-multiprocess.c b/content/chapters/app-interact/lab/solution/password-cracker/password-cracker-multiprocess.c index f1a8773734..e187e22a22 100644 --- a/content/chapters/app-interact/lab/solution/password-cracker/password-cracker-multiprocess.c +++ b/content/chapters/app-interact/lab/solution/password-cracker/password-cracker-multiprocess.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multiprocess.c b/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multiprocess.c index e8675fa11c..187456f744 100644 --- a/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multiprocess.c +++ b/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multiprocess.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multithread.c b/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multithread.c index 21d9c2abad..6dc11f90ee 100644 --- a/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multithread.c +++ b/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multithread.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lab/support/time-server/client.c b/content/chapters/app-interact/lab/support/time-server/client.c index af65be2f46..056796a9a4 100644 --- a/content/chapters/app-interact/lab/support/time-server/client.c +++ b/content/chapters/app-interact/lab/support/time-server/client.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lab/support/time-server/server.c b/content/chapters/app-interact/lab/support/time-server/server.c index dc7318c473..696f464cdf 100644 --- a/content/chapters/app-interact/lab/support/time-server/server.c +++ b/content/chapters/app-interact/lab/support/time-server/server.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/reader.c b/content/chapters/app-interact/lecture/demo/comm-channels/reader.c index d774a60925..9e6b45e650 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/reader.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/reader.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/receive_fifo.c b/content/chapters/app-interact/lecture/demo/comm-channels/receive_fifo.c index 4e3b2be898..cec29b48a2 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/receive_fifo.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/receive_fifo.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/receive_net_dgram_socket.c b/content/chapters/app-interact/lecture/demo/comm-channels/receive_net_dgram_socket.c index 717bc69772..c78aca6971 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/receive_net_dgram_socket.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/receive_net_dgram_socket.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/receive_net_stream_socket.c b/content/chapters/app-interact/lecture/demo/comm-channels/receive_net_stream_socket.c index 1647ab2c5f..fde0e96bfb 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/receive_net_stream_socket.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/receive_net_stream_socket.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/receive_unix_socket.c b/content/chapters/app-interact/lecture/demo/comm-channels/receive_unix_socket.c index 4c6738bab4..87116b8288 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/receive_unix_socket.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/receive_unix_socket.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/send_fifo.c b/content/chapters/app-interact/lecture/demo/comm-channels/send_fifo.c index 938beeb808..4e4524dbcb 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/send_fifo.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/send_fifo.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/send_net_dgram_socket.c b/content/chapters/app-interact/lecture/demo/comm-channels/send_net_dgram_socket.c index 0e69d79144..094916865a 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/send_net_dgram_socket.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/send_net_dgram_socket.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/send_net_stream_socket.c b/content/chapters/app-interact/lecture/demo/comm-channels/send_net_stream_socket.c index b14eb0a7c2..36126b69a0 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/send_net_stream_socket.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/send_net_stream_socket.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/send_receive_pipe.c b/content/chapters/app-interact/lecture/demo/comm-channels/send_receive_pipe.c index 57de8255cf..a27cfdb08f 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/send_receive_pipe.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/send_receive_pipe.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/send_unix_socket.c b/content/chapters/app-interact/lecture/demo/comm-channels/send_unix_socket.c index 32d42fadf1..82de753d64 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/send_unix_socket.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/send_unix_socket.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/comm-channels/writer.c b/content/chapters/app-interact/lecture/demo/comm-channels/writer.c index b2050f20e7..89a6b1df48 100644 --- a/content/chapters/app-interact/lecture/demo/comm-channels/writer.c +++ b/content/chapters/app-interact/lecture/demo/comm-channels/writer.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/fibonacci-server/connection.c b/content/chapters/app-interact/lecture/demo/fibonacci-server/connection.c index 5725de8262..1474b25e1a 100644 --- a/content/chapters/app-interact/lecture/demo/fibonacci-server/connection.c +++ b/content/chapters/app-interact/lecture/demo/fibonacci-server/connection.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_pool_server.c b/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_pool_server.c index ff5a749120..c77326342a 100644 --- a/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_pool_server.c +++ b/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_pool_server.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_pool_server_works.c b/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_pool_server_works.c index 10ee324e44..c2fe4ff348 100644 --- a/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_pool_server_works.c +++ b/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_pool_server_works.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_server.c b/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_server.c index 16715bc0dd..007090d9b9 100644 --- a/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_server.c +++ b/content/chapters/app-interact/lecture/demo/fibonacci-server/mp_server.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/fibonacci-server/mt_pool_server.c b/content/chapters/app-interact/lecture/demo/fibonacci-server/mt_pool_server.c index 339bbee423..590c0769b6 100644 --- a/content/chapters/app-interact/lecture/demo/fibonacci-server/mt_pool_server.c +++ b/content/chapters/app-interact/lecture/demo/fibonacci-server/mt_pool_server.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/fibonacci-server/mt_server.c b/content/chapters/app-interact/lecture/demo/fibonacci-server/mt_server.c index 8ef0f776a0..edf69cfd45 100644 --- a/content/chapters/app-interact/lecture/demo/fibonacci-server/mt_server.c +++ b/content/chapters/app-interact/lecture/demo/fibonacci-server/mt_server.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/fibonacci-server/server.c b/content/chapters/app-interact/lecture/demo/fibonacci-server/server.c index 86f74eb975..983e9e754b 100644 --- a/content/chapters/app-interact/lecture/demo/fibonacci-server/server.c +++ b/content/chapters/app-interact/lecture/demo/fibonacci-server/server.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/fibonacci-server/task.c b/content/chapters/app-interact/lecture/demo/fibonacci-server/task.c index 49cdacea2b..eb23c2b856 100644 --- a/content/chapters/app-interact/lecture/demo/fibonacci-server/task.c +++ b/content/chapters/app-interact/lecture/demo/fibonacci-server/task.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/interrupt/rt_signal_printer.c b/content/chapters/app-interact/lecture/demo/interrupt/rt_signal_printer.c index f1efb33c9c..f1ea66e26d 100644 --- a/content/chapters/app-interact/lecture/demo/interrupt/rt_signal_printer.c +++ b/content/chapters/app-interact/lecture/demo/interrupt/rt_signal_printer.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/interrupt/rt_signal_sender.c b/content/chapters/app-interact/lecture/demo/interrupt/rt_signal_sender.c index 9d7b094b98..eb1743a62c 100644 --- a/content/chapters/app-interact/lecture/demo/interrupt/rt_signal_sender.c +++ b/content/chapters/app-interact/lecture/demo/interrupt/rt_signal_sender.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/interrupt/signal_printer.c b/content/chapters/app-interact/lecture/demo/interrupt/signal_printer.c index 7fac166d36..55f601cf47 100644 --- a/content/chapters/app-interact/lecture/demo/interrupt/signal_printer.c +++ b/content/chapters/app-interact/lecture/demo/interrupt/signal_printer.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/interrupt/signal_sender.c b/content/chapters/app-interact/lecture/demo/interrupt/signal_sender.c index bd8aa1a095..8d16c5dd03 100644 --- a/content/chapters/app-interact/lecture/demo/interrupt/signal_sender.c +++ b/content/chapters/app-interact/lecture/demo/interrupt/signal_sender.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/interrupt/signal_sender_sleep.c b/content/chapters/app-interact/lecture/demo/interrupt/signal_sender_sleep.c index 397fb120c2..93ece4bb38 100644 --- a/content/chapters/app-interact/lecture/demo/interrupt/signal_sender_sleep.c +++ b/content/chapters/app-interact/lecture/demo/interrupt/signal_sender_sleep.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/lock/proc_flock.c b/content/chapters/app-interact/lecture/demo/lock/proc_flock.c index 898eed0596..c431acdf1a 100644 --- a/content/chapters/app-interact/lecture/demo/lock/proc_flock.c +++ b/content/chapters/app-interact/lecture/demo/lock/proc_flock.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/lock/proc_sem.c b/content/chapters/app-interact/lecture/demo/lock/proc_sem.c index 0e18bc4f24..29c9f7fb0d 100644 --- a/content/chapters/app-interact/lecture/demo/lock/proc_sem.c +++ b/content/chapters/app-interact/lecture/demo/lock/proc_sem.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/lock/thread_mutex.c b/content/chapters/app-interact/lecture/demo/lock/thread_mutex.c index 13b5ae162d..0911e70cd7 100644 --- a/content/chapters/app-interact/lecture/demo/lock/thread_mutex.c +++ b/content/chapters/app-interact/lecture/demo/lock/thread_mutex.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/lock/thread_sem.c b/content/chapters/app-interact/lecture/demo/lock/thread_sem.c index 28f8d03a96..9c75408092 100644 --- a/content/chapters/app-interact/lecture/demo/lock/thread_sem.c +++ b/content/chapters/app-interact/lecture/demo/lock/thread_sem.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/shared-mem/shmem_reader.c b/content/chapters/app-interact/lecture/demo/shared-mem/shmem_reader.c index 9f3d2d5302..515b272175 100644 --- a/content/chapters/app-interact/lecture/demo/shared-mem/shmem_reader.c +++ b/content/chapters/app-interact/lecture/demo/shared-mem/shmem_reader.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/shared-mem/shmem_threads.c b/content/chapters/app-interact/lecture/demo/shared-mem/shmem_threads.c index ac42904aa3..e0110bd6eb 100644 --- a/content/chapters/app-interact/lecture/demo/shared-mem/shmem_threads.c +++ b/content/chapters/app-interact/lecture/demo/shared-mem/shmem_threads.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/shared-mem/shmem_writer.c b/content/chapters/app-interact/lecture/demo/shared-mem/shmem_writer.c index 19de7632cb..815d5eec42 100644 --- a/content/chapters/app-interact/lecture/demo/shared-mem/shmem_writer.c +++ b/content/chapters/app-interact/lecture/demo/shared-mem/shmem_writer.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/sync/proc_sem_first.c b/content/chapters/app-interact/lecture/demo/sync/proc_sem_first.c index e4290c194c..41570e4d1a 100644 --- a/content/chapters/app-interact/lecture/demo/sync/proc_sem_first.c +++ b/content/chapters/app-interact/lecture/demo/sync/proc_sem_first.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/sync/proc_sem_second.c b/content/chapters/app-interact/lecture/demo/sync/proc_sem_second.c index 05b802e9ea..b8e3048b2c 100644 --- a/content/chapters/app-interact/lecture/demo/sync/proc_sem_second.c +++ b/content/chapters/app-interact/lecture/demo/sync/proc_sem_second.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/sync/proc_sig_first.c b/content/chapters/app-interact/lecture/demo/sync/proc_sig_first.c index bcfcd9de6e..6ccd2c9736 100644 --- a/content/chapters/app-interact/lecture/demo/sync/proc_sig_first.c +++ b/content/chapters/app-interact/lecture/demo/sync/proc_sig_first.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/sync/proc_sig_second.c b/content/chapters/app-interact/lecture/demo/sync/proc_sig_second.c index 553bd39427..0f76712684 100644 --- a/content/chapters/app-interact/lecture/demo/sync/proc_sig_second.c +++ b/content/chapters/app-interact/lecture/demo/sync/proc_sig_second.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/sync/thread_cond.c b/content/chapters/app-interact/lecture/demo/sync/thread_cond.c index 7b409997da..abc562d5c1 100644 --- a/content/chapters/app-interact/lecture/demo/sync/thread_cond.c +++ b/content/chapters/app-interact/lecture/demo/sync/thread_cond.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/app-interact/lecture/demo/sync/thread_sem.c b/content/chapters/app-interact/lecture/demo/sync/thread_sem.c index 590ac4bf37..0d22bcab71 100644 --- a/content/chapters/app-interact/lecture/demo/sync/thread_sem.c +++ b/content/chapters/app-interact/lecture/demo/sync/thread_sem.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/compute/.gitignore b/content/chapters/compute/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/content/chapters/compute/lab/content/arena.md b/content/chapters/compute/lab/content/arena.md deleted file mode 100644 index 39f227c7eb..0000000000 --- a/content/chapters/compute/lab/content/arena.md +++ /dev/null @@ -1,296 +0,0 @@ -# Arena - -## Threads and Processes: `clone` - -Let's go back to our initial demos that used threads and processes. -We'll see that in order to create both threads and processes, the underlying Linux syscall is `clone`. -For this, we'll run both `sum_array_threads` and `sum_array_processes` under `strace`. -As we've already established, we're only interested in the `clone` syscall: - -```console -student@os:~/.../lab/support/sum-array/c$ strace -e clone,clone3 ./sum_array_threads 2 -clone(child_stack=0x7f60b56482b0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[1819693], tls=0x7f60b5649640, child_tidptr=0x7f60b5649910) = 1819693 -clone(child_stack=0x7f60b4e472b0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[1819694], tls=0x7f60b4e48640, child_tidptr=0x7f60b4e48910) = 1819694 - -student@os:~/.../lab/support/sum-array/c$ strace -e clone,clone3 ./sum_array_processes 2 -clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f7a4e346650) = 1820599 -clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f7a4e346650) = 1820600 -``` - -We ran each program with an argument of 2, so we have 2 calls to `clone`. -Notice that in the case of threads, the `clone3 syscall receives more arguments. -The relevant flags passed as arguments when creating threads are documented in [`clone`'s man page](https://man.archlinux.org/man/clone3.2.en): - -- `CLONE_VM`: the child and the parent process share the same VAS -- `CLONE_{FS,FILES,SIGHAND}`: the new thread shares the filesystem information, file and signal handlers with the one that created it -The syscall also receives valid pointers to the new thread's stack and TLS, i.e. the only parts of the VAS that are distinct between threads (although they are technically accessible from all threads). - -By contrast, when creating a new process, the arguments of the `clone` syscall are simpler (i.e. fewer flags are present). -Remember that in both cases `clone` creates a new **thread**. -When creating a process, `clone` creates this new thread within a new separate address space. - -## Libraries for Parallel Processing - -In `support/sum-array/c/sum_array_threads.c` we spawned threads "manually" by using the `pthread_create()` function. -This is **not** a syscall, but a wrapper over the common syscall used by both `fork()` (which is also not a syscall) and `pthread_create()`. - -Still, `pthread_create()` is not yet a syscall. -In order to see what syscall `pthread_create()` uses, check out [this section at the end of the lab](#threads-and-processes-clone). - -Most programming languages provide a more advanced API for handling parallel computation. - -### `std.parallelism` in D - -D language's standard library exposes the [`std.parallelism`](https://dlang.org/phobos/std_parallelism.html), which provides a series of parallel processing functions. -One such function is `reduce()`, which splits an array between a given number of threads and applies a given operation to these chunks. -In our case, the operation simply adds the elements to an accumulator: `a + b`. -Follow and run the code in `support/sum-array/d/sum_array_threads_reduce.d`. - -The number of threads is used within a [`TaskPool`](https://dlang.org/phobos/std_parallelism.html#.TaskPool). -This structure is a thread manager (not scheduler). -It silently creates the number of threads we request and then `reduce()` spreads its workload between these threads. - -### OpenMP for C - -Unlike D, C does not support parallel computation by design. -It needs a library to do advanced things, like `reduce()` from D. -We have chosen to use the OpenMP library for this. -Follow the code in `support/sum-array/c/sum_array_threads_openmp.c`. - -The `#pragma` used in the code instructs the compiler to enable the `omp` module, and to parallelise the code. -In this case, we instruct the compiler to perform a reduce of the array, using the `+` operator, and to store the results in the `result` variable. -This reduction uses threads to calculate the sum, similar to `summ_array_threads.c`, but in a much more optimised form. - -Now compile and run the `sum_array_threads_openmp` binary using 1, 2, 4, and 8 threads as before. -You'll see lower running times than `sum_array_threads` due to the highly-optimised code emitted by the compiler. -For this reason and because library functions are usually much better tested than your own code, it is always preferred to use a library function for a given task. - -## Shared Memory - -As you remember from the [Data chapter](../../../data/), one way to allocate a given number of pages is to use the `mmap()` syscall. -Let's look at its [man page](https://man7.org/linux/man-pages/man2/mmap.2.html), specifically at the `flags` argument. -Its main purpose is to determine the way in which child processes interact with the mapped pages. - -[Quiz](../quiz/mmap-cow-flag.md) - -Now let's test this flag, as well as its opposite: `MAP_SHARED`. -Compile and run the code in `support/shared-memory/shared_memory.c`. - -1. See the value read by the parent is different from that written by the child. -Modify the `flags` parameter of `mmap()` so they are the same. - -1. Create a semaphore in the shared page and use it to make the parent signal the child before it can exit. -Use the API defined in [`semaphore.h`](https://man7.org/linux/man-pages/man0/semaphore.h.0p.html). -**Be careful!** -The value written and read previously by the child and the parent, respectively, must not change. - -One way of creating a shared semaphore is to place it within a shared memory area, as we've just done. -This only works between "related" processes. -If you want to share a semaphore or other types of memory between any two processes, you need filesystem support. -For this, you should use **named semaphores**, created using [`sem_open()`](https://man7.org/linux/man-pages/man3/sem_open.3.html). -You'll get more accustomed to such functions in the [Application Interaction chapter](../../../app-interact/). - -## Mini-shell - -### First Step: `system` Dissected - -You already know that `system` calls `fork()` and `execve()` to create the new process. -Let's see how and why. -First, we run the following command to trace the `execve()` syscalls used by `sleepy_creator`. -We'll leave `fork()` for later. - -```console -student@os:~/.../support/sleepy$ strace -e execve -ff -o syscalls ./sleepy_creator -``` - -At this point, you will get two files whose names start with `syscalls`, followed by some numbers. -Those numbers are the PIDs of the parent and the child process. -Therefore, the file with the higher number contains logs of the `execve` and `clone` syscalls issued by the parent process, while -the other logs those two syscalls when made by the child process. -Let's take a look at them. -The numbers below will differ from those on your system: - -```console -student@os:~/.../support/sleepy:$ cat syscalls.2523393 # syscalls from parent process -execve("sleepy_creator", ["sleepy_creator"], 0x7ffd2c157758 /* 39 vars */) = 0 ---- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=2523394, si_uid=1052093, si_status=0, si_utime=0, si_stime=0} --- -+++ exited with 0 +++ - -student@os:~/.../support/sleepy:$ cat syscalls.2523394 # syscalls from child process -execve("/bin/sh", ["sh", "-c", "sleep 10"], 0x7ffd36253be8 /* 39 vars */) = 0 -execve("/usr/bin/sleep", ["sleep", "10"], 0x560f41659d40 /* 38 vars */) = 0 -+++ exited with 0 +++ -``` - -[Quiz](../quiz/who-calls-execve-parent.md) - -Now notice that the child process doesn't simply call `execve("/usr/bin/sleep" ...)`. -It first changes its virtual address space (VAS) to that of a `bash` process (`execve("/bin/sh" ...)`) and then that `bash` process switches its VAS to `sleep`. -Therefore, calling `system()` is equivalent to running `` in the command-line. - -With this knowledge in mind, let's implement our own mini-shell. -Start from the skeleton code in `support/mini-shell/mini_shell.c`. -We're already running our Bash interpreter from the command-line, so there's no need to `exec` another Bash from it. -Simply `exec` the command. - -[Quiz](../quiz/mini-shell-stops-after-command.md) - -So we need a way to "save" the `mini_shell` process before `exec()`-ing our command. -Find a way to do this. - -> **Hint**: You can see what `sleepy` does and draw inspiration from there. -> Use `strace` to also list the calls to `clone()` performed by `sleepy` or its children. -> [Remember](#threads-and-processes-clone) what `clone()` is used for and use its parameters to deduce which of the two scenarios happens to `sleepy`. - -**Moral of the story**: We can add another step to the moral of [our previous story](./processes.md#practice-fork). -When spawning a new command, the call order is: - -- parent: `fork()`, `exec()`, `wait()` -- child: `exit()` - -### Command Executor in Another language - -Now implement the same functionality (a Bash command executor) in any other language, other than C/C++. -Use whatever API is provided by your language of choice for creating and waiting for processes. - -## The GIL - -Throughout this lab, you might have noticed that there were no thread exercises _in Python_. -If you did, you probably wondered why. -It's not because Python does not support threads, because it does, but because of a mechanism called the **Global Interpreter Lock**, or GIL. -As its name suggests, this is a lock implemented inside most commonly used Python interpreter (CPython), which only allows **one** thread to run at a given time. -As a consequence, multithreaded programs written in Python run **concurrently**, not in parallel. -For this reason, you will see no speedup even when you run an embarrassingly parallel code in parallel. - -However, keep in mind that this drawback does not make threads useless in Python. -They are still useful and widely used when a process needs to perform many IO-bound tasks (i.e.: tasks that involve many file reads / writes or network requests). -Such tasks run many blocking syscalls that require the thread to switch from the RUNNING state to WAITING. -Doing so voluntarily makes threads viable because they rarely run for their entire time slice and spend most of the time waiting for data. -So it doesn't hurt them to run concurrently, instead of in parallel. - -Do not make the confusion to believe threads in Python are [user-level threads](./scheduling.md#user-level-vs-kernel-level-threads). -[`threading.Thread`](https://docs.python.org/3/library/threading.html#threading.Thread)s are kernel-level threads. -It's just that they are forced to run concurrently by the GIL. - -### Practice: Array Sum in Python - -Let's first probe this by implementing two parallel versions of the code in `support/sum-array/python/sum_array_sequential.py`. -One version should use threads and the other should use processes. -Run each of them using 1, 2, 4, and 8 threads / processes respectively and compare the running times. -Notice that the running times of the multithreaded implementation do not decrease. -This is because the GIL makes it so that those threads that you create essentially run sequentially. - -The GIL also makes it so that individual Python instructions are atomic. -Run the code in `support/race-condition/python/race_condition.py`. -Every time, `var` will be 0 because the GIL doesn't allow the two threads to run in parallel and reach the critical section at the same time. -This means that the instructions `var += 1` and `var -= 1` become atomic. - -### But Why? - -Unlike Bigfoot, or the Loch Ness monster, we have proof that the GIL is real. -At first glance, this seems like a huge disadvantage. -Why force threads to run sequentially? -The answer has to do with memory management. -In the [Data chapter](../../../data), you learned that one way of managing memory is via _garbage collection_ (GC). -In Python, the GC uses reference counting, i.e. each object also stores the number of live pointers to it (variables that reference it). -You can see that this number needs to be modified atomically by the interpreter to avoid race conditions. -This involves adding locks to **all** Python data structures. -This large number of locks doesn't scale for a language as large and open as Python. -In addition, it also introduces the risk of _deadlocks_. -You can read more on this topic [in this article](https://realpython.com/python-gil/) and if you think eliminating the GIL looks like an easy task, which should have been done long ago, check the requirements [here](https://wiki.python.org/moin/GlobalInterpreterLock). -They're not trivial to meet. - -Single-threaded-ness is a common trope for interpreted languages to use some sort of GIL. -[Ruby MRI, the reference Ruby interpreter](https://git.ruby-lang.org/ruby.git), uses a similar mechanism, called the [Global VM Lock](https://ivoanjo.me/blog/2022/07/17/tracing-ruby-global-vm-lock/). -JavaScript is even more straightforward: it is single-threaded by design, also for GC-related reasons. -It does, however, support asynchronous actions, but these are executed on the same thread as every other code. -This is implemented by placing each instruction on a [call stack](https://medium.com/swlh/what-does-it-mean-by-javascript-is-single-threaded-language-f4130645d8a9). - -## Atomic Assembly - -No, this section is not about nukes, sadly :(. -Instead, we aim to get accustomed to the way in which the x86 ISA provides atomic instructions. - -This mechanism looks very simple. -It is but **one instruction prefix**: `lock`. -It is not an instruction with its own separate opcode, but a prefix that slightly modifies the opcode of the following instructions to make the CPU execute it atomically (i.e. with exclusive access to the data bus). - -`lock` must only be place before an instruction that executes a _read-modify-write_ action. -For example, we cannot place it before a `mov` instruction, as the action of a `mov` is simply `read` or `write`. -Instead, we can place it in front of an `inc` instruction if its operand is memory. - -Look at the code in `support/race-condition/asm/race_condition_lock.S`. -It's an Assembly equivalent of the code you've already seen many times so far (such as `support/race-condition/c/race_condition.c`). -Assemble and run it a few times. -Notice the different results you get. - -Now add the `lock` prefix before `inc` and `dec`. -Reassemble and rerun the code. -And now we have synchronized the two threads by leveraging CPU support. - -## Synchronization - Thread-Safe Data Structure - -Now it's time for a fully practical exercise. -Go to `support/CLIST/`. -In the file `clist.c` you'll find a simple implementation of an array list. -Although correct, it is not (yet) thread-safe. - -The code in `test.c` verifies its single-threaded correctness, while the one in `test_parallel.c` verifies it works properly with multiple threads. -Your task is to synchronize this data structure using whichever primitives you like. -Try to keep the implementation efficient. -Aim to decrease your running times as much as you can. - -## Minor and Major Page Faults - -The code in `support/page-faults/page_faults.c` generates some minor and major page faults. -Open 2 terminals: one in which you will run the program, and one which will monitor the page faults of the program. -In the monitoring terminal, run the following command: - -```console -watch -n 1 'ps -eo min_flt,maj_flt,cmd | grep ./page_faults | head -n 1' -``` - -Compile the program and run it in the other terminal. -You must press `enter` one time, before the program will prompt you to press `enter` more times. -Watch the first number on the monitoring terminal; -it increases. -Those are the minor page faults. - -### Minor Page Faults - -A minor page fault is generated whenever a requested page is present in the physical memory, as a frame, but that frame isn't allocated to the process generating the request. -These types of page faults are the most common, and they happen when calling functions from dynamic libraries, allocating heap memory, loading programs, reading files that have been cached, and many more situations. -Now back to the program. - -The monitoring command already starts with some minor page faults, generated when loading the program. - -After pressing `enter`, the number increases, because a function from a dynamic library (libc) is fetched when the first `printf()` is executed. -Subsequent calls to functions that are in the same memory page as `printf()` won't generate other page faults. - -After allocating the 100 Bytes, you might not see the number of page faults increase. -This is because the "bookkeeping" data allocated by `malloc()` was able to fit in an already mapped page. -The second allocation, the 1GB one, will always gnereate one minor page fault - for the bookkeeping data about the allocated memory zone. -Notice that not all the pages for the 1GB are allocated. -They are allocated - and generate page faults - when modified. -By now you should know that this mechanism is called [copy-on-write](copy-on-write.md). - -Continue with pressing `enter` and observing the effects util you reach opening `file.txt`. - -Note that neither opening a file, getting information about it, nor mapping it in memory using `mmap()`, generate page faults. -Also note the `posix_fadvise()` call after the one to `fstat()`. -With this call we force the OS to not cache the file, so we can generate a **major page fault**. - -### Major Page Faults - -Major page faults happen when a page is requested, but it isn't present in the physical memory. -These types of page faults happen in 2 situations: - -* a page that was swapped out (to the disk), due to lack of memory, is now accessed - this case is harder to show -* the OS needs to read a file from the disk, because the file contents aren't present in the cache - the case we are showing now - -Press `enter` to print the file contents. -Note the second number go up in the monitoring terminal. - -Comment the `posix_fadvise()` call, recompile the program, and run it again. -You won't get any major page fault, because the file contents are cached by the OS, to avoid those page faults. -As a rule, the OS will avoid major page faults whenever possible, because they are very costly in terms of running time. diff --git a/content/chapters/compute/lab/content/overview.md b/content/chapters/compute/lab/content/overview.md deleted file mode 100644 index c64a6f0915..0000000000 --- a/content/chapters/compute/lab/content/overview.md +++ /dev/null @@ -1,12 +0,0 @@ -# Compute - -## Contents - -- [Hardware Perspective](./hardware-perspective.md) -- [Processes](./processes.md) -- [Threads](./threads.md) -- [Processes and Threads in `apache2`](./processes-threads-apache2.md) -- [Copy-on-Write](./copy-on-write.md) -- [Synchronization](./synchronization.md) -- [User Level Threads](./user-level-threads.md) -- [Arena](./arena.md) diff --git a/content/chapters/compute/lab/content/processes.md b/content/chapters/compute/lab/content/processes.md deleted file mode 100644 index 0e3d918a50..0000000000 --- a/content/chapters/compute/lab/content/processes.md +++ /dev/null @@ -1,276 +0,0 @@ -# Processes - -A process is simply a running program. -Let's take the `ls` command as a trivial example. -`ls` is a **program** on your system. -It has a binary file which you can find and inspect with the help of the `which` command: - -```console -student@os:~$ which ls -/usr/bin/ls - -student@os:~$ file /usr/bin/ls -/usr/bin/ls: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=6e3da6f0bc36b6398b8651bbc2e08831a21a90da, for GNU/Linux 3.2.0, stripped -``` - -When you run it, the `ls` binary stored **on the disk** at `/usr/bin/ls` is read by another application called the **loader**. -The loader spawns a **process** by copying some of the contents `/usr/bin/ls` in memory (such as the `.text`, `.rodata` and `.data` sections). -Using `strace`, we can see the [`execve`](https://man7.org/linux/man-pages/man2/execve.2.html) system call: - -```console -student@os:~$ strace -s 100 ls -a # -s 100 limits strings to 100 bytes instead of the default 32 -execve("/usr/bin/ls", ["ls", "-a"], 0x7fffa7e0d008 /* 61 vars */) = 0 -[...] -write(1, ". .. content\tCONTRIBUTING.md COPYING.md .git .gitignore README.md REVIEWING.md\n", 86. .. content CONTRIBUTING.md COPYING.md .git .gitignore README.md REVIEWING.md -) = 86 -close(1) = 0 -close(2) = 0 -exit_group(0) = ? -+++ exited with 0 +++ -``` - -Look at its parameters: - -- the path to the **program**: `/usr/bin/ls` -- the list of arguments: `"ls", "-a"` -- the environment variables: the rest of the syscall's arguments - -`execve` invokes the loader to load the VAS of the `ls` process **by replacing that of the existing process**. -All subsequent syscalls are performed by the newly spawned `ls` process. -We will get into more details regarding `execve` [towards the end of this lab](./arena.md#first-step-system-dissected). - -![Loading of `ls` Process](../media/loading-of-ls-process.svg) - -## Sum of the Elements in an Array - -Let's assume we only have one process on our system, and that process knows how to add the numbers in an array. -It can use however many resources it wants, since there is no other process to contest it. -It would probably look like the code in `support/sum-array/c/sum_array_sequential.c`. -The program also measures the time spent computing the sum. -Let's compile and run it: - -```console -student@os:~/.../lab/support/sum-array/c$ ./sum_array_sequential -Array sum is: 49945994146 -Time spent: 127 ms -``` - -You will most likely get a different sum (because the array is made up of random numbers) and a different time than the ones shown above. -This is perfectly fine. -Use these examples qualitatively, not quantitatively. - -## Spreading the Work Among Other Processes - -Due to how it's implemented so far, our program only uses one of our CPU's cores. -We never tell it to distribute its workload to other cores. -This is wasteful as the rest of our cores remain unused: - -```console -student@os:~$ lscpu | grep ^CPU\(s\): -CPU(s): 8 -``` - -We have 7 more cores waiting to add numbers in our array. - -![What if we used 100% of the CPU?](../media/100-percent-cpu.jpeg) - -What if we use 7 more processes and spread the task of adding the numbers in this array between them? -If we split the array into several equal parts and designate a separate process to calculate the sum of each part, we should get a speedup because now the work performed by each individual process is reduced. - -Let's take it methodically. -Compile and run `sum_array_processes.c` using 1, 2, 4 and 8 processes respectively. -If your system only has 4 cores ([hyperthreading](https://www.intel.com/content/www/us/en/gaming/resources/hyper-threading.html) included), limit your runs to 4 processes. -Note the running times for each number of processes. -We expect the speedups compared to our reference run to be 1, 2, 4 and 8 respectively, right? - -[Quiz](../quiz/processes-speedup.md) - -You most likely did get some speedup, especially when using 8 processes. -Now we will try to improve this speedup by using **threads** instead. - -Also notice that we're not using hundreds or thousands of processes. -Assuming our system has 8 cores, only 8 _threads_ (we'll see this later in the lab) can run at the same time. -In general, **the maximum number of threads that can run at the same time is equal to the number of cores**. -In our example, each process only has one thread: its main thread. -So by consequence and by forcing the terminology (because it's the main thread of these processes that is running, not the processes themselves), we can only run in parallel a number of processes equal to at most the number of cores. - -### Practice: Baby steps - Python - -Run the code in `support/create-process/popen.py`. -It simply spawns a new process running the `ls` command using [`subprocess.Popen()`](https://docs.python.org/3/library/subprocess.html#subprocess.Popen). -Do not worry about the huge list of arguments that `Popen()` takes. -They are used for **inter-process-communication**. -You'll learn more about this in the [Application Interaction chapter](../../../app-interact/). - -Note that this usage of `Popen()` is not entirely correct. -You'll discover why in the next exercise, but for now focus on simply understanding how to use `Popen()` on its own. - -Now change the command to anything you want. -Also give it some arguments. -From the outside, it's as if you were running these commands from the terminal. - -### Practice: High level - Python - -Head over to `support/sleepy/sleepy_creator.py`. -Use `subprocess.Popen()` to spawn 10 `sleep 1000` processes. - -1. Solve `TODO 1`: use `subprocess.Popen()` to spawn 10 `sleep 1000` processes. - - Start the script: - - ```console - student@os:~/.../lab/support/sleepy$ python3 sleepy_creator.py - ``` - - Look for the parent process: - - ```console - student@os:~$ ps -e -H -o pid,ppid,cmd | (head -1; grep "python3 sleepy_creator.py") - ``` - - It is a `python3` process, as this is the interpreter that runs the script, but we call it the `sleepy_creator.py` process for simplicity. - No output will be provided by the above command, as the parent process (`sleepy_creator.py`) dies before its child processes (the 10 `sleep 1000` subprocesses) finish their execution. - The parent process of the newly created child processes is an `init`-like process: either `systemd`/`init` or another system process that adopts orphan processes. - Look for the `sleep` child processes using: - - ```console - student@os:~$ ps -e -H -o pid,ppid,cmd | (head -1; grep sleep) - PID PPID CMD - 4164 1680 sleep 1000 - 4165 1680 sleep 1000 - 4166 1680 sleep 1000 - 4167 1680 sleep 1000 - 4168 1680 sleep 1000 - 4169 1680 sleep 1000 - 4170 1680 sleep 1000 - 4171 1680 sleep 1000 - 4172 1680 sleep 1000 - 4173 1680 sleep 1000 - ``` - - Notice that the child processes do not have `sleepy_creator.py` as a parent. - What's more, as you saw above, `sleepy_creator.py` doesn't even exist anymore. - The child processes have been adopted by an `init`-like process (in the output above, that process has PID `1680` - `PPID` stands for _parent process ID_). - - [Quiz](../quiz/parent-of-sleep-processes.md) - -1. Solve `TODO 2`: change the code in `sleepy_creator.py` so that the `sleep 1000` processes remain the children of `sleepy_creator.py`. - This means that the parent / creator process must **not** exit until its children have finished their execution. - In other words, the parent / creator process must **wait** for the termination of its children. - Check out [`Popen.wait()`](https://docs.python.org/3/library/subprocess.html#subprocess.Popen.wait) and add the code that makes the parent / creator process wait for its children. - Before anything, terminate the `sleep` processes created above: - - ```console - student@os:~$ pkill sleep - ``` - - Start the program, again, as you did before: - - ```console - student@os:~/.../lab/support/sleepy$ python3 sleepy_creator.py - ``` - - On another terminal, verify that `sleepy_creator.py` remains the parent of the `sleep` processes it creates: - - ```console - student@os:~$ ps -e -H -o pid,ppid,cmd | (head -1; grep sleep) - PID PPID CMD - 16107 9855 python3 sleepy_creator.py - 16108 16107 sleep 1000 - 16109 16107 sleep 1000 - 16110 16107 sleep 1000 - 16111 16107 sleep 1000 - 16112 16107 sleep 1000 - 16113 16107 sleep 1000 - 16114 16107 sleep 1000 - 16115 16107 sleep 1000 - 16116 16107 sleep 1000 - 16117 16107 sleep 1000 - ``` - - Note that the parent process `sleepy_creator.py` (`PID 16107`) is still alive, and its child processes (the 10 `sleep 1000`) have its ID as their `PPID`. - You've successfully waited for the child processes to finish their execution. - -### Practice: Lower level - C - -Now let's see how to create a child process in C. -There are multiple ways of doing this. -For now, we'll start with a higher-level approach. - -Go to `support/sleepy/sleepy_creator.c` and use [`system`](https://man7.org/linux/man-pages/man3/system.3.html) to create a `sleep 1000` process. - -[Quiz](../quiz/create-sleepy-process-ending.md) - -The `man` page also mentions that `system` calls `fork()` and `exec()` to run the command it's given. -If you want to find out more about them, head over to the [Arena and create your own mini-shell](./arena.md#mini-shell). - -### Practice: Wait for Me - -Run the code in `support/wait-for-me/wait_for_me_processes.py`. -The parent process creates one child that writes and message to the given file. -Then the parent reads that message. -Simple enough, right? -But running the code raises a `FileNotFoundError`. -If you inspect the file you gave the script as an argument, it does contain a string. -What's going on? - -[Quiz](../quiz/cause-of-file-not-found-error.md) - -In order to solve race conditions, we need **synchronization**. -This is a mechanism similar to a set of traffic lights in a crossroads. -Just like traffic lights allow some cars to pass only after others have already passed, synchronization is a means for threads to communicate with each other and tell each other to access a resource or not. - -The most basic form of synchronization is **waiting**. -Concretely, if the parent process **waits** for the child to end, we are sure the file is created and its contents are written. -Use `join()` to make the parent wait for its child before reading the file. - -### Practice: `fork()` - -Up to now we've been creating processes using various high-level APIs, such as `Popen()`, `Process()` and `system()`. -Yes, despite being a C function, as you've seen from its man page, `system()` itself calls 2 other functions: `fork()` to create a process and `execve()` to execute the given command. -As you already know from the [Software Stack](../../../software-stack/) chapter, library functions may call one or more underlying system calls or other functions. -Now we will move one step lower on the call stack and call `fork()` ourselves. - -`fork()` creates one child process that is _almost_ identical to its parent. -We say that `fork()` returns **twice**: once in the parent process and once more in the child process. -This means that after `fork()` returns, assuming no error has occurred, both the child and the parent resume execution from the same place: the instruction following the call to `fork()`. -What's different between the two processes is the value returned by `fork()`: - -- **child process**: `fork()` returns 0 -- **parent process**: `fork()` returns the PID of the child process (> 0) -- **on error**: `fork()` returns -1, only once, in the initial process - -Therefore, the typical code for handling a `fork()` is available in `support/create-process/fork.c`. -Take a look at it and then run it. -Notice what each of the two processes prints: - -- the PID of the child is also known by the parent -- the PPID of the child is the PID of the parent - -Unlike `system()`, who also waits for its child, when using `fork()` we must do the waiting ourselves. -In order to wait for a process to end, we use the [`waitpid()`](https://linux.die.net/man/2/waitpid) syscall. -It places the exit code of the child process in the `status` parameter. -This argument is actually a bit-field containing more information than merely the exit code. -To retrieve the exit code, we use the `WEXITSTATUS` macro. -Keep in mind that `WEXITSTATUS` only makes sense if `WIFEXITED` is true, i.e. if the child process finished on its own and wasn't killed by another one or by an illegal action (such as a segfault or illegal instruction) for example. -Otherwise, `WEXITSTATUS` will return something meaningless. -You can view the rest of the information stored in the `status` bit-field [in the man page](https://linux.die.net/man/2/waitpid). - -Now modify the example to do the following: - -1. Change the return value of the child process so that the value displayed by the parent is changed. - -1. Create a child process of the newly created child. -Use a similar logic and a similar set of prints to those in the support code. -Take a look at the printed PIDs. -Make sure the PPID of the "grandchild" is the PID of the child, whose PPID is, in turn, the PID of the parent. - -**Moral of the story**: Usually the execution flow is: - -1. `fork()`, followed by - -1. `wait()` (called by the parent) - -1. `exit()`, called by the child. - -The order of last 2 steps may be swapped. diff --git a/content/chapters/compute/lab/content/threads.md b/content/chapters/compute/lab/content/threads.md deleted file mode 100644 index 32f94682e5..0000000000 --- a/content/chapters/compute/lab/content/threads.md +++ /dev/null @@ -1,150 +0,0 @@ -# Threads - -## Spreading the Work Among Other Threads - -Compile the code in `support/sum-array/c/sum_array_threads.c` and run it using 1, 2, 4 and 8 threads as you did before. -Each thread runs the `calculateArrayPartSum` function and then finishes. -Running times should be _slightly_ smaller than the implementation using processes. -This slight time difference is caused by process creation actions, which are costlier than thread creation actions. -Because a process needs a separate virtual address space (VAS) and needs to duplicate some internal structures such as the file descriptor table and page table, it takes the operating system more time to create it than to create a thread. -On the other hand, threads belonging to the same process share the same VAS and, implicitly, the same OS-internal structures. -Therefore, they are more lightweight than processes. - -### Practice: Wait for Me Once More - -Go to `support/wait-for-me/wait_for_me_threads.c`. -Spawn a thread that executes the `negate_array()` function. -For now, do not wait for it to finish; -simply start it. - -Compile the code and run the resulting executable several times. -See that the negative numbers appear from different indices. -This is precisely the nondeterminism that we talked about [in the previous section](./processes.md#practice-wait-for-me). - -Now wait for that thread to finish and see that all the printed numbers are consistently negative. - -As you can see, waiting is a very coarse form of synchronization. -If we only use waiting, we can expect no speedup as a result of parallelism, because one thread must finish completely before another can continue. -We will discuss more fine-grained synchronization mechanisms [later in this lab](./synchronization.md). - -Also, at this point, you might be wondering why this exercise is written in D, while [the same exercise, but with processes](./processes.md#practice-wait-for-me) was written in Python. -There is a very good reason for this and has to do with how threads are synchronized by default in Python. -You can find out what this is about [in the Arena section](./arena.md#the-gil), after you have completed the [Synchronization section](./synchronization.md). - -## Threads vs Processes - -So why use the implementation that spawns more processes if it's slower than the one using threads? -The table below lists the differences between threads and processes. -Generally, if we only want to do some computing, we use threads. -If we need to drastically change the behaviour of the program, we need a new program altogether, or we need more than computing (e.g. communication on the network to create a computing cluster), we use processes. - -| PROCESS | THREAD | -| :--------------------------------------------------- | :---------------------------------------------------------------------- | -| independent | part of a process | -| collection of threads | shares VAS with other threads | -| slower creation (new page table must be created) | faster creation | -| longer context switch duration (TLB must be flushed) | shorter context switch duration (part of the same process, so same TLB) | -| ending means ending all threads | other threads continue when finished | - -### Safety - -Compile and run the two programs in `support/sum-array-bugs/seg-fault/`, first with 2 processes and threads and then with 4. -They do the same thing as before: compute the sum of the elements in an array, but with a twist: each of them contains a bug causing a segfault. -Notice that `sum_array_threads` doesn't print anything with 4 threads, but merely a "Segmentation fault" message. -On the other hand, `sum_array_processes` prints a sum and a running time, albeit different from the sums we've seen so far. - -The reason is that signals such as `SIGSEGV`, which is used when a segmentation fault happens affect the entire process that handles them. -Therefore, when we split our workload between several threads and one of them causes an error such as a segfault, that error is going to terminate the entire process. -The same thing happens when we use processes instead of threads: one process causes an error, which gets it killed, but the other processes continue their work unhindered. -This is why we end up with a lower sum in the end: because one process died too early and didn't manage to write the partial sum it had computed to the `results` array. - -### Practice: Wait for It - -The process that spawns all the others and subsequently calls `waitpid` to wait for them to finish can also get their return codes. -Update the code in `support/sum-array-bugs/seg-fault/sum_array_processes.c` and modify the call to `waitpid` to obtain and investigate this return code. -Display an appropriate message if one of the child processes returns an error. - -Remember to use the appropriate [macros](https://linux.die.net/man/2/waitpid) for handling the `status` variable that is modified by `waitpid()`, as it is a bit-field. -When a process runs into a system error, it receives a signal. -A signal is a means to interrupt the normal execution of a program from the outside. -It is associated with a number. -Use `kill -l` to find the full list of signals. - -[Quiz](../quiz/seg-fault-exit-code.md) - -So up to this point we've seen that one advantage of processes is that they offer better safety than threads. -Because they use separate virtual address spaces, sibling processes are better isolated than threads. -Thus, an application that uses processes can be more robust to errors than if it were using threads. - -### Memory Corruption - -Because they share the same address space, threads run the risk of corrupting each other's data. -Take a look at the code in `support/sum-array-bugs/memory-corruption/python/`. -The two programs only differ in how they spread their workload. -One uses threads while the other uses processes. - -Run both programs with and without memory corruption. -Pass any value as a third argument to trigger the corruption. - -```console -student@os:~/.../sum-array-bugs/memory-corruption/python$ python3 memory_corruption_processes.py # no memory corruption -[...] - -student@os:~/.../sum-array-bugs/memory-corruption/python$ python3 memory_corruption_processes.py 1 # do memory corruption -[...] -``` - -The one using threads will most likely print a negative sum, while the other displays the correct sum. -This happens because all threads refer to the same memory for the array `arr`. -What happens to the processes is a bit more complicated. - -[Later in this lab](./copy-on-write.md), we will see that initially, the page tables of all processes point to the same physical frames or `arr`. -When the malicious process tries to corrupt this array by **writing data to it**, the OS duplicates the original frames of `arr` so that the malicious process writes the corrupted values to these new frames, while leaving the original ones untouched. -This mechanism is called **Copy-on-Write** and is an OS optimisation so that memory is shared between the parent and the child process, until one of them attempts to write to it. -At this point, this process receives its own separate copies of the previously shared frames. - -Note that in order for the processes to share the `sums` dictionary, it is not created as a regular dictionary, but using the `Manager` module. -This module provides some special data structures that are allocated in **shared memory** so that all processes can access them. -You can learn more about shared memory and its various implementations [in the Arena section](./arena.md#shared-memory). - -## Memory Layout of Multithreaded Programs - -When a new thread is created, a new stack is allocated for a thread. -The default stack size if `8 MB` / `8192 KB`: - -```console -student@os:~$ ulimit -s -8192 -``` - -Enter the `support/multithreaded/` directory to observe the update of the memory layout when creating new threads. - -Build the `multithreaded` executable: - -```console -student@os:~/.../lab/support/multithreaded$ make -``` - -Start the program: - -```console -student@os:~/.../lab/support/multithreaded$ ./multithreaded -Press key to start creating threads ... -[...] -``` - -And investigate it with `pmap` on another console, while pressing a key to create new threads. - -As you can see, there is a new `8192 KB` area created for every thread, also increasing the total virtual size. - -### Practice - -1. Build the multithreaded program as a static executable by adding `LDFLAGS = -static` to the Makefile: - Run it. - You can check the executable is statically linked by executing the command `ldd multithreaded`. - Notice the same effect of the thread creation on the process memory layout: the creation of a new stack of `8192 KB`. - -1. Make a program in another language of your choice that creates threads. - Investigate it with `pmap`. - -[Quiz](../quiz/thread-memory.md) diff --git a/content/chapters/compute/lab/content/user-level-threads.md b/content/chapters/compute/lab/content/user-level-threads.md deleted file mode 100644 index 9911909cfc..0000000000 --- a/content/chapters/compute/lab/content/user-level-threads.md +++ /dev/null @@ -1,107 +0,0 @@ -# User-Level Threads - -User-level threads differ from the threads you are used to (kernel-level threads, those created by pthread_create). -This kind of threads are scheduled by an user-level scheduler, and can run on the same kernel-level thread. -From now on, we will reffer to user-level threads as fibers, and kernel-level threads as simply threads. - -We will use the fiber implementation from libboost. -This implementation uses a cooperative scheduler on each thread, meaning that each fiber has to yield, in order for other fiber to be executed. -We will also use C++, and the standard `thread` implementation. - -## Prerequisites - -Unless you are using the OS docker image, you will need to install `cmake` and `libboost`. -You can do this with the following command: - -```console -student@os:~$ sudo apt-get install cmake libboost-context-dev libboost-fiber-dev -``` - -## Creation - -Follow the `support/user-level-threads/simple.cc` implementation. -It creates `NUM_FIBERS` fibers, that each prints "Hello World". -To compile and run the program, do the following steps: - -```console -student@os:~/.../lab/support/user-level-threads$ mkdir build/ -student@os:~/.../lab/support/user-level-threads$ cd build/ -student@os:~/.../lab/support/user-level-threads$ cmake -S .. -B . -student@os:~/.../lab/support/user-level-threads$ make -student@os:~/.../lab/support/user-level-threads$ ./simple -``` - -The `cmake` step must be executed only once. -After modifying the source files, it is enough to run `make`. - -### Practice: Sleeper Fiber - -Add in `support/user-level-threads/simple.cc` a fiber that sleeps for 5 seconds, before the other ones are created. -What happens? -Answer in this [quiz](../quiz/sleeping-on-a-fiber.md). - -## No system calls - -Use `strace` to find calls to `clone()` in the execution of `simple`. -Can you find any? -Provide your answer in this [quiz](../quiz/fiber-strace.md) -Remember that `clone()` is the system call used to create **kernel-level** threads, as pointed out [here](./arena.md#threads-and-processes-clone). - -## Synchronization - -By default, the fibers that run on the same thread are synchronized - no race-conditions can occur. -This is illustrated by the `support/user-level-threads/sum.cc` implementation. - -The user can, however, implement further synchronization, by using the `yield()` call, or classic synchronization methods, like mutexes, barriers and condition variables. - -### Yielding - -As the scheduler is cooperative, each fiber can yield (or not), to allow another fiber to run. -Follow the `support/user-level-threads/yield_launch.cc` implementation and run it. -Note the `boost::fibers::launch::dispatch` parameter provided to the fiber constructor. -It notifies the scheduler to start the fibre as soon as it is created. -In order to explain the output, we must consider that the fibers are created by a **main fiber**, that is scheduled along with the others, in this case. - -#### Practice - -Modify the launch parameter into `boost::fibers::launch::post`, compile and notice the differences. -The `post` parameter notifies the scheduler not to start the fibers imediately, but rather place them into an execution queue. -Their execution will start after the main fiber calls the `join()` function. - -### Barriers - -Follow the `support/user-level-threads/yield_barrier.cc` implementation. -It uses a barrier to achieve the same result as the previos implementation, that used `post` as the launch parameter. - -## Interaction Between Threads and Fibers - -As we mentioned before, multiple fibers can run on the same thread, and a scheduler is implemented on each thread. -By default, the scheduling algorithm is [`round_robin`](https://www.guru99.com/round-robin-scheduling-example.html). -It runs the fibers, in the order of their creation, until they yield or finish their work. -If a fiber yields, it is placed at the back of the round-robin queue. -Using this scheduler, each thread only uses its fibers; -if one thread has more work to do than another, bad luck. -This may lead to starvation. - -But there are other scheduler implementations, such as `shared_work` and `work_stealing`. -Follow the `support/user-level-threads/threads_and_fibers.cc` implementation. -It creates multiple fibers and threads, and uses the `shared_work` scheduler to balance the workload between the threads. -Each main fiber, from each thread, is suspended until all worker fibers have completed their work, using a condition variable. - -```cpp -cnd_count.wait( lk, [](){ return 0 == fiber_count; } ); -``` - -The program also uses `thread local storage` and `fiber local storage` to store the ID of each thread / fiber. - -Now change the `shared_work` scheduler into the `work_stealing` one. -It takes a parameter, the number of threads that will use that scheduler. - -Compile, rerun and note the differences. -The `work_stealing` scheduler, as the name suggests, will "steal" fibers from other schedulers. -So, if the `shared_work` scheduler tried to balance the available work between the available threads, the `work_stealing` one will focus on having as many threads as possible on 100% workload. -Vary the number of threads and fibers, and the workload (maybe put each fibre to do some computational-intensive work), and observe the results. - -### C++ unique_lock - -`unique_lock` is a type of mutex that is unlocked automatically when the end of its scope is reached (end of function or bracket-pair). diff --git a/content/chapters/compute/lab/quiz/state-of-new-ult.md b/content/chapters/compute/lab/quiz/state-of-new-ult.md deleted file mode 100644 index 2b5cad620e..0000000000 --- a/content/chapters/compute/lab/quiz/state-of-new-ult.md +++ /dev/null @@ -1,16 +0,0 @@ -# State of new ULT - -## Question Text - -What is the first state that is assigned to a newly created ULT? - -## Question Answers - -- RUNNING -- READY -- COMPLETED - -## Feedback - -Inside the function `threads_create()`, we can see the following snippet `queue_enqueue(ready, new)`. -Each new thread is thus added to the READY queue. diff --git a/content/chapters/compute/lab/solution/apache2/get_docroot.sh b/content/chapters/compute/lab/solution/apache2/get_docroot.sh deleted file mode 100755 index 072ffc2c8d..0000000000 --- a/content/chapters/compute/lab/solution/apache2/get_docroot.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Cleanup. -rm -f out.* -docker exec apache2-test pkill strace - -sleep 1 - -# Get pids of worker processes. -pids=$(docker exec apache2-test ps -ef | grep httpd | awk '{ print $2 }') - -# We don't know which worker will handle our request, so attach strace to all of them. -for pid in $pids; do - docker exec apache2-test strace -f -p $pid &> out.$pid & -done - -sleep 1 - -# Do a request. -curl -s localhost:8080 > /dev/null - -# Search for an open call on index.html. -s=$(grep 'open.*index.html' out.*) - -if [ $? -eq 0 ]; then - echo "Found line: $s" - - indexhtml="$(grep -o '"[^"]\+"' <<< $s)" - - # strip quotes - indexhtml=${indexhtml%\"} - indexhtml=${indexhtml#\"} - - docroot="$(dirname $indexhtml)" - echo "Document root is $docroot" -else - echo "Couldn't find the open syscall" -fi - -# Cleanup. -rm -f out.* -docker exec apache2-test pkill strace diff --git a/content/chapters/compute/lab/solution/create-process/Makefile b/content/chapters/compute/lab/solution/create-process/Makefile deleted file mode 120000 index 298e4ebabb..0000000000 --- a/content/chapters/compute/lab/solution/create-process/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../support/create-process/Makefile \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/libult/CPPLINT.cfg b/content/chapters/compute/lab/solution/libult/CPPLINT.cfg deleted file mode 120000 index af4ff7b0a3..0000000000 --- a/content/chapters/compute/lab/solution/libult/CPPLINT.cfg +++ /dev/null @@ -1 +0,0 @@ -../../support/libult/CPPLINT.cfg \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/libult/Makefile b/content/chapters/compute/lab/solution/libult/Makefile deleted file mode 120000 index 021dfe1d60..0000000000 --- a/content/chapters/compute/lab/solution/libult/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../support/libult/Makefile \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/libult/queue.c b/content/chapters/compute/lab/solution/libult/queue.c deleted file mode 120000 index 38387a797a..0000000000 --- a/content/chapters/compute/lab/solution/libult/queue.c +++ /dev/null @@ -1 +0,0 @@ -../../support/libult/queue.c \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/libult/queue.h b/content/chapters/compute/lab/solution/libult/queue.h deleted file mode 120000 index 35b1c34d53..0000000000 --- a/content/chapters/compute/lab/solution/libult/queue.h +++ /dev/null @@ -1 +0,0 @@ -../../support/libult/queue.h \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/libult/tcb.c b/content/chapters/compute/lab/solution/libult/tcb.c deleted file mode 120000 index 0856c61b50..0000000000 --- a/content/chapters/compute/lab/solution/libult/tcb.c +++ /dev/null @@ -1 +0,0 @@ -../../support/libult/tcb.c \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/libult/tcb.h b/content/chapters/compute/lab/solution/libult/tcb.h deleted file mode 120000 index 3efce45155..0000000000 --- a/content/chapters/compute/lab/solution/libult/tcb.h +++ /dev/null @@ -1 +0,0 @@ -../../support/libult/tcb.h \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/libult/threads.h b/content/chapters/compute/lab/solution/libult/threads.h deleted file mode 120000 index c5566f8ba3..0000000000 --- a/content/chapters/compute/lab/solution/libult/threads.h +++ /dev/null @@ -1 +0,0 @@ -../../support/libult/threads.h \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/mini-shell/.gitignore b/content/chapters/compute/lab/solution/mini-shell/.gitignore deleted file mode 100644 index fb647f35e4..0000000000 --- a/content/chapters/compute/lab/solution/mini-shell/.gitignore +++ /dev/null @@ -1 +0,0 @@ -mini_shell diff --git a/content/chapters/compute/lab/solution/mini-shell/Makefile b/content/chapters/compute/lab/solution/mini-shell/Makefile deleted file mode 120000 index 170c601782..0000000000 --- a/content/chapters/compute/lab/solution/mini-shell/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../support/mini-shell/Makefile \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/race-condition/asm/Makefile b/content/chapters/compute/lab/solution/race-condition/asm/Makefile deleted file mode 120000 index c1cc8d02ce..0000000000 --- a/content/chapters/compute/lab/solution/race-condition/asm/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../../support/race-condition/asm/Makefile \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/race-condition/c/Makefile b/content/chapters/compute/lab/solution/race-condition/c/Makefile deleted file mode 120000 index b3f5a0ec79..0000000000 --- a/content/chapters/compute/lab/solution/race-condition/c/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../../support/race-condition/c/Makefile \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/shared-memory/Makefile b/content/chapters/compute/lab/solution/shared-memory/Makefile deleted file mode 120000 index dc783d5b07..0000000000 --- a/content/chapters/compute/lab/solution/shared-memory/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../support/shared-memory/Makefile \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/sleepy/Makefile b/content/chapters/compute/lab/solution/sleepy/Makefile deleted file mode 120000 index dd77d85d9e..0000000000 --- a/content/chapters/compute/lab/solution/sleepy/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../support/sleepy/Makefile \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/sum-array-bugs/seg-fault/Makefile b/content/chapters/compute/lab/solution/sum-array-bugs/seg-fault/Makefile deleted file mode 120000 index 570bfaaaa1..0000000000 --- a/content/chapters/compute/lab/solution/sum-array-bugs/seg-fault/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../../support/sum-array-bugs/seg-fault/Makefile \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/sum-array-bugs/seg-fault/generate_random_array.d b/content/chapters/compute/lab/solution/sum-array-bugs/seg-fault/generate_random_array.d deleted file mode 120000 index 1e04b91714..0000000000 --- a/content/chapters/compute/lab/solution/sum-array-bugs/seg-fault/generate_random_array.d +++ /dev/null @@ -1 +0,0 @@ -../../../support/sum-array/d/generate_random_array.d \ No newline at end of file diff --git a/content/chapters/compute/lab/solution/wait-for-me/Makefile b/content/chapters/compute/lab/solution/wait-for-me/Makefile deleted file mode 120000 index 2524d3961d..0000000000 --- a/content/chapters/compute/lab/solution/wait-for-me/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../support/wait-for-me/Makefile \ No newline at end of file diff --git a/content/chapters/compute/lab/support/CLIST/.gitignore b/content/chapters/compute/lab/support/CLIST/.gitignore deleted file mode 100644 index 9daeafb986..0000000000 --- a/content/chapters/compute/lab/support/CLIST/.gitignore +++ /dev/null @@ -1 +0,0 @@ -test diff --git a/content/chapters/compute/lab/support/CLIST/Makefile b/content/chapters/compute/lab/support/CLIST/Makefile deleted file mode 100644 index 0cfb09f624..0000000000 --- a/content/chapters/compute/lab/support/CLIST/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -TARGET = test -TARGET_PARALLEL = test_parallel - -all: $(TARGET) $(TARGET_PARALLEL) - -include ../../../../../common/makefile/linux.mk - -LIB_OBJECTS = clist.o -LIBRARY = libclist.a -TEST_OBJECTS = test.o $(LIBRARY) -TEST_OBJECTS_PARALLEL = test_parallel.o $(LIBRARY) - -LDLIBS = -lpthread - -$(LIBRARY): $(LIB_OBJECTS) - ar rcs $(LIBRARY) $(LIB_OBJECTS) - -$(TARGET): $(TEST_OBJECTS) - $(CC) $^ -o $@ $(LDLIBS) - -$(TARGET_PARALLEL): $(TEST_OBJECTS_PARALLEL) - $(CC) $^ -o $@ $(LDLIBS) - -clean:: - -rm -f $(LIBRARY) $(TARGET) $(TARGET_PARALLEL) diff --git a/content/chapters/compute/lab/support/CLIST/clist.c b/content/chapters/compute/lab/support/CLIST/clist.c deleted file mode 100644 index 9933ad5beb..0000000000 --- a/content/chapters/compute/lab/support/CLIST/clist.c +++ /dev/null @@ -1,427 +0,0 @@ -/******************************************/ -/* */ -/* Alexander Agdgomlishvili */ -/* */ -/* cdevelopment@mail.com */ -/* */ -/******************************************/ - -/** - * BSD 2-Clause License - * - * Copyright (c) 2020, Alexander Agdgomlishvili - * cdevelopment@mail.com - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* Github link: https://github.com/AlexanderAgd/CLIST */ - -#include -#include -#include -#include -#include -#include "clist.h" - -typedef struct -{ - int count; /* Number of items in the list. */ - int alloc_size; /* Allocated size in quantity of items */ - int lastSearchPos; /* Position of last search - firstMatch or LastMatch */ - size_t item_size; /* Size of each item in bytes. */ - void *items; /* Pointer to the list */ -} CList_priv_; - -int CList_Realloc_(CList *l, int n) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - if (n < p->count) - { - fprintf(stderr, "CList: ERROR! Can not realloc to '%i' size - count is '%i'\n", n, p->count); - assert(n >= p->count); - return 0; - } - - if (n == 0 && p->alloc_size == 0) - n = 2; - - void *ptr = realloc(p->items, p->item_size * n); - if (ptr == NULL) - { - fprintf(stderr, "CList: ERROR! Can not reallocate memory!\n"); - return 0; - } - p->items = ptr; - p->alloc_size = n; - return 1; -} - -void *CList_Add_(CList *l, void *o) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - if (p->count == p->alloc_size && - CList_Realloc_(l, p->alloc_size * 2) == 0) - return NULL; - - char *data = (char*) p->items; - data = data + p->count * p->item_size; - memcpy(data, o, p->item_size); - p->count++; - return data; -} - -void *CList_Insert_(CList *l, void *o, int n) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - if (n < 0 || n > p->count) - { - fprintf(stderr, "CList: ERROR! Insert position outside range - %d; n - %d.\n", - p->count, n); - assert(n >= 0 && n <= p->count); - return NULL; - } - - if (p->count == p->alloc_size && - CList_Realloc_(l, p->alloc_size * 2) == 0) - return NULL; - - size_t step = p->item_size; - char *data = (char*) p->items + n * step; - memmove(data + step, data, (p->count - n) * step); - memcpy(data, o, step); - p->count++; - return data; -} - -void *CList_Replace_(CList *l, void *o, int n) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - if (n < 0 || n >= p->count) - { - fprintf(stderr, "CList: ERROR! Replace position outside range - %d; n - %d.\n", - p->count, n); - assert(n >= 0 && n < p->count); - return NULL; - } - - char *data = (char*) p->items; - data = data + n * p->item_size; - memcpy(data, o, p->item_size); - return data; -} - -void CList_Remove_(CList *l, int n) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - if (n < 0 || n >= p->count) - { - fprintf(stderr, "CList: ERROR! Remove position outside range - %d; n - %d.\n", - p->count, n); - assert(n >= 0 && n < p->count); - return; - } - - size_t step = p->item_size; - char *data = (char*)p->items + n * step; - memmove(data, data + step, (p->count - n - 1) * step); - p->count--; - - if (p->alloc_size > 3 * p->count && p->alloc_size >= 4) /* Dont hold much memory */ - CList_Realloc_(l, p->alloc_size / 2); -} - -void *CList_At_(CList *l, int n) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - if (n < 0 || n >= p->count) - { - fprintf(stderr, "CList: ERROR! Get position outside range - %d; n - %d.\n", - p->count, n); - assert(n >= 0 && n < p->count); - return NULL; - } - - char *data = (char*) p->items; - data = data + n * p->item_size; - return data; -} - -void *CList_firstMatch_(CList *l, const void *o, size_t shift, size_t size, int string) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - char *data = (char*) p->items; - size_t step = p->item_size; - p->lastSearchPos = -1; - - if (shift + size > p->item_size) - { - fprintf(stderr, "CList: ERROR! Wrong ranges for firstMatch - " - "shift '%zu', size '%zu', item_size '%zu'\n", shift, size, p->item_size); - assert(shift + size <= p->item_size); - return NULL; - } - - if (shift == 0 && size == 0) - size = p->item_size; - - size_t i = shift; - size_t end = p->count * step; - int index = 0; - - if (string) - { - for (; i < end; i += step, index++) - { - if (strncmp(data + i, o, size) == 0) - { - p->lastSearchPos = index; - return (data + i - shift); - } - } - } - else - { - for (; i < end; i += step, index++) - { - if (memcmp(data + i, o, size) == 0) - { - p->lastSearchPos = index; - return (data + i - shift); - } - } - } - - return NULL; -} - -void *CList_lastMatch_(struct CList *l, const void *o, size_t shift, size_t size, int string) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - char *data = (char*) p->items; - size_t step = p->item_size; - p->lastSearchPos = -1; - - if (shift + size > p->item_size) - { - fprintf(stderr, "CList: ERROR! Wrong ranges for lastMatch - " - "shift '%zu', size '%zu', item_size '%zu'\n", shift, size, p->item_size); - assert(shift + size <= p->item_size); - return NULL; - } - - if (shift == 0 && size == 0) - size = p->item_size; - - int index = p->count - 1; - long i = index * step + shift; - if (string) - { - for (; i >= 0; i -= step, index--) - { - if (strncmp(data + i, o, size) == 0) - { - p->lastSearchPos = index; - return (data + i - shift); - } - } - } - else - { - for (; i >= 0; i -= step, index--) - { - if (memcmp(data + i, o, size) == 0) - { - p->lastSearchPos = index; - return (data + i - shift); - } - } - } - - return NULL; -} - -int CList_index_(CList *l) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - return p->lastSearchPos; -} - -int CList_swap_(CList *l, int a, int b) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - - if (a < 0 || a >= p->count || b < 0 || b >= p->count) - { - fprintf(stderr, "CList: ERROR! Swap position outside range - %i, %i; count - %d.\n", - a, b, p->count); - assert(a >= 0 && a < p->count && b >= 0 && b < p->count); - return 0; - } - - if (a == b) return 1; /* ? Good ? :D */ - - char *data = (char*) p->items; - size_t step = p->item_size; - - if (p->count == p->alloc_size && - CList_Realloc_(l, p->alloc_size + 1) == 0) - return 0; - - memcpy(data + p->count * step, data + a * step, step); - memcpy(data + a * step, data + b * step, step); - memcpy(data + b * step, data + p->count * step, step); - return 1; -} - -int CList_Count_(CList *l) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - return p->count; -} - -int CList_AllocSize_(CList *l) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - return p->alloc_size; -} - -size_t CList_ItemSize_(CList *l) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - return p->item_size; -} - -void CList_Clear_(CList *l) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - free(p->items); - p->items = NULL; - p->alloc_size = 0; - p->count = 0; -} - -void CList_Free_(CList *l) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - free(p->items); - free(p); - free(l); - l = NULL; -} - -void CList_print_(CList *l, size_t shift, int n, const char *type) -{ - CList_priv_ *p = (CList_priv_*) l->priv; - - if (shift >= p->item_size) - { - fprintf(stderr, "CList: ERROR! Wrong shift value for list print - " - "shift '%zu', item_size '%zu'\n", shift, p->item_size); - assert(shift < p->item_size); - return; - } - - printf("\nCList: count = %i item_size = %zu alloc_size = %i type = %s\n", - p->count, p->item_size, p->alloc_size, type); - - if (n > 0) - { - int tp = -1; - if (type == NULL) tp = 0; /* Print out pointers */ - else if (strcmp(type, "char") == 0) tp = 1; - else if (strcmp(type, "short") == 0) tp = 2; - else if (strcmp(type, "int") == 0) tp = 3; - else if (strcmp(type, "long") == 0) tp = 4; - else if (strcmp(type, "uintptr_t") == 0) tp = 5; - else if (strcmp(type, "size_t") == 0) tp = 6; - else if (strcmp(type, "double") == 0) tp = 7; - else if (strcmp(type, "string") == 0) tp = 8; - - if (tp == -1) - { - fprintf(stderr, "CList: Can not print - not supported type - %s\n\n", type); - return; - } - - n = (n > p->count) ? p->count : n; - char *data = (char*) p->items + shift; - size_t step = p->item_size; - int i = 0; - for (; i < n; i++) - { - switch (tp) - { - case 0: printf("%p ", data); break; - case 1: printf("%c ", *(char*) data); break; - case 2: printf("%hi ", *(short*) data); break; - case 3: printf("%i ", *(int*) data); break; - case 4: printf("%li ", *(long*) data); break; - case 5: printf("0x%lx ", *(uintptr_t*) data); break; - case 6: printf("%zu ", *(size_t*) data); break; - case 7: printf("%f ", *(double*) data); break; - case 8: printf("%s\n", data); break; - default: return; - } - - data += step; - } - printf("\n\n"); - } -} - -CList *CList_init(size_t objSize) -{ - CList *lst = malloc(sizeof(CList)); - CList_priv_ *p = malloc(sizeof(CList_priv_)); - if (!lst || !p) - { - fprintf(stderr, "CList: ERROR! Can not allocate CList!\n"); - return NULL; - } - p->count = 0; - p->alloc_size = 0; - p->lastSearchPos = -1; - p->item_size = objSize; - p->items = NULL; - lst->add = &CList_Add_; - lst->insert = &CList_Insert_; - lst->replace = &CList_Replace_; - lst->remove = &CList_Remove_; - lst->at = &CList_At_; - lst->realloc = &CList_Realloc_; - lst->count = &CList_Count_; - lst->firstMatch = &CList_firstMatch_; - lst->lastMatch = &CList_lastMatch_; - lst->index = &CList_index_; - lst->swap = &CList_swap_; - lst->allocSize = &CList_AllocSize_; - lst->itemSize = &CList_ItemSize_; - lst->print = &CList_print_; - lst->clear = &CList_Clear_; - lst->free = &CList_Free_; - lst->priv = p; - return lst; -} - diff --git a/content/chapters/compute/lab/support/CLIST/test.c b/content/chapters/compute/lab/support/CLIST/test.c deleted file mode 100644 index 1c518530cb..0000000000 --- a/content/chapters/compute/lab/support/CLIST/test.c +++ /dev/null @@ -1,375 +0,0 @@ -/******************************************/ -/* */ -/* Alexander Agdgomlishvili */ -/* */ -/* cdevelopment@mail.com */ -/* */ -/******************************************/ - -/** - * BSD 2-Clause License - * - * Copyright (c) 2020, Alexander Agdgomlishvili - * cdevelopment@mail.com - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* Github link: https://github.com/AlexanderAgd/CLIST */ - -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) -#include -#endif - -#if defined(_POSIX_VERSION) -#include -#elif defined(WIN32) -#include -#endif - -#include -#include -#include -#include -#include -#include "clist.h" - -#if defined(WIN32) -int gettimeofday(struct timeval *tp, void *tzp); -#endif - -size_t diff_usec(struct timeval start, struct timeval end) -{ - return (1000000 * (end.tv_sec - start.tv_sec) + end.tv_usec - start.tv_usec); -} - -int main(void) -{ - void *obj = NULL; - int i = 0; - int n = 0; - - /******************************************************* CHAR LIST */ - - n = sizeof(char); - CList *lst = CList_init(n); - - for (i = 33; i < 123; i++) - { - obj = &i; - lst->add(lst, obj); /* Adding obj items to the end of array */ - } - lst->print(lst, 0, 150, "char"); - - n = 32; - char ch = 'A'; - obj = lst->at(lst, n); /* Get 32nd item of array */ - printf("Position %i contains symbol \'%c\'\n", n, *((char*)obj)); - - ch = '!'; - obj = &ch; - lst->firstMatch(lst, obj, 0, 0, 0); /* Find first match of '!' char */ - /* You can skip (set to zero) "shift" and "size" parameters when compare whole item */ - n = lst->index(lst); - printf("Index of \'%c\' is %i\n", ch, n); - - ch = '+'; - lst->firstMatch(lst, obj, 0, 0, 0); - n = lst->index(lst); - printf("Index of \'%c\' is %i\n", ch, n); - - n = 0; - for (i = 15; i > 0; i--) - lst->remove(lst, 0); /* Remove item at index 0 from array 15 times */ - - lst->print(lst, 0, 150, "char"); - lst->free(lst); - - /******************************************************* SHORT INT LIST */ - - n = sizeof(short); - lst = CList_init(n); /* Always set object size you will work with */ - - short sh = 1001; - for (i = 0; i < 24; i++, sh += 1000) - { - obj = &sh; - lst->add(lst, obj); - } - - lst->print(lst, 0, 100, "short"); - - sh = 5001; - lst->insert(lst, &sh, 20); /* Insert value of 'sh' to position 20 */ - - lst->insert(lst, &sh, 25); /* Insert value of 'sh' to position 25 */ - lst->print(lst, 0, lst->count(lst), "short"); - - lst->lastMatch(lst, &sh, 0, 0, 0); /* Find last match of '5001' short */ - n = lst->index(lst); - printf("Last index of \'%i\' is %i\n", sh, n); - - while (n != -1) - { - lst->firstMatch(lst, &sh, 0, 0, 0); /* Find first match of '5001' short */ - n = lst->index(lst); - - if (n != -1) /* Remove it from the list */ - lst->remove(lst, n); - } - - sh = 1111; - lst->replace(lst, &sh, 0); /* Replace object at position '0' */ - lst->print(lst, 0, 100, "short"); - - lst->clear(lst); /* CLear list */ - lst->free(lst); - - /******************************************************* STRUCT LIST */ - - struct sample - { - long int l; - double d; - int s; - char info[24]; - union u - { - long long lli; - char c; - } u; - union u *dat; - } sample; - - n = sizeof(sample); - lst = CList_init(n); - - printf("Size of struct 'sample' = %zu bytes\n", sizeof(sample)); - - n = 0; - for (i = 0; i < 8; i++) /* Let create list of a struct */ - { - struct sample sam = - { - .l = 10 * i, - .d = 5.010101 * i, - .s = i, - .info = "", - .u = { .lli = 11 * i }, - .dat = &(sam.u) - }; - - sprintf(sam.info, "Some string_%c", 80 + i); - - obj = &sam; - lst->insert(lst, obj, n); /* Insert each object at index '0' */ - } - - lst->print(lst, 0, 20, NULL); - - int j; - int count = lst->count(lst); - for (j = 0; j < 6; j++, i = 0) /* Print out struct content */ - { - for (i = 0; i < count; i++) - { - struct sample *sam = lst->at(lst, i); - switch (j) - { - case 0: printf("%15li ", sam->l); break; - case 1: printf("%15lf ", sam->d); break; - case 2: printf("%15i ", sam->s); break; - case 3: printf("%15s ", sam->info); break; - case 4: printf("%15lli ", sam->u.lli); break; - case 5: printf("%15p ", sam->dat); break; - default: break; - } - } - printf("\n\n"); - } - - /* Advanced usage of firstMatch and lastMatch */ - - size_t shift = offsetof(struct sample, info); - size_t size = sizeof(sample.info); - int its_a_string = 1; - struct sample *smpl = lst->firstMatch(lst, "Some string_R", shift, size, its_a_string); - n = lst->index(lst); - - printf("List item with index %i contains string \"%s\"\n", n, smpl->info); - - long long int longValue = 55; - shift = offsetof(struct sample, u); - size = sizeof(long long int); - int string = 0; - smpl = lst->firstMatch(lst, &longValue, shift, size, string); - n = lst->index(lst); - - printf("List item with index %i contains long long int \"%lli\"\n", n, smpl->u.lli); - - /* Print struct member */ - - shift = offsetof(struct sample, info); - lst->print(lst, shift, 8, "string"); - lst->free(lst); - - /******************************************************* POINTERS LIST */ - - /* If we want create list of objects located in diffent places of memory, */ - /* let create pointers list or in our case "uintptr_t" address list */ - - printf("Size of 'uintptr_t' = %zu bytes\n", sizeof(uintptr_t)); - - n = sizeof(uintptr_t); - lst = CList_init(n); - - struct sample sm1 = { 64, 6.4, 4, "ABC company", { 16 }, obj }; - struct sample sm2 = { 128, 12.8, 8, "Discovery", { 1024 }, (void*)&sh }; /* Just some sample data */ - - uintptr_t addr = (uintptr_t) &sm1; /* Cast reference to address value */ - lst->add(lst, &addr); - - addr = (uintptr_t) &sm2; - lst->add(lst, &addr); - - struct sample *sm3 = malloc(sizeof(sample)); - sm3->l = 256; - sm3->d = 25.6; - sm3->s = 16; - strcpy(sm3->info, "TV show"); - sm3->u.lli = 2048; - sm3->dat = (void*) &sm2; - - addr = (uintptr_t) sm3; - lst->add(lst, &addr); - - lst->print(lst, 0, 20, "uintptr_t"); - - count = lst->count(lst); - for (j = 0; j < 6; j++, i = 0) /* Print out struct content */ - { - for (i = 0; i < count; i++) - { - uintptr_t *ad = lst->at(lst, i); - struct sample *sam = (struct sample*) *ad; /* Cast address value to struct pointer */ - switch (j) - { - case 0: printf("%15li ", sam->l); break; - case 1: printf("%15lf ", sam->d); break; - case 2: printf("%15i ", sam->s); break; - case 3: printf("%15s ", sam->info); break; - case 4: printf("%15lli ", sam->u.lli); break; - case 5: printf("%15p ", sam->dat); break; - default: break; - } - } - printf("\n"); - } - - printf("\n"); - //lst->print(lst, shift, 3, "uintptr_t"); - free(sm3); - lst->free(lst); - - /******************************************************* PERFOMANCE TEST */ - - n = sizeof(int); - lst = CList_init(n); - - size_t time; - i = 0; - n = 10000; - struct timeval start; - struct timeval end; - - printf("PERFOMANCE TEST - 1 second contains 1000000 microseconds\n\n"); - - gettimeofday(&start, NULL); - for(i=0; i < n; i++) - lst->add(lst, &i); - gettimeofday(&end, NULL); - time = diff_usec(start, end); - printf("Add of %i int takes - %zu microseconds\n", n, time); - - gettimeofday(&start, NULL); - for(i=0; i < n; i++) - lst->remove(lst, 0); - gettimeofday(&end, NULL); - time = diff_usec(start, end); - printf("Remove from position '0' of %i int takes - %zu microseconds\n", n, time); - - gettimeofday(&start, NULL); - for(i=0; i < n; i++) - lst->insert(lst, &i, 0); - gettimeofday(&end, NULL); - time = diff_usec(start, end); - printf("Insert to position '0' of %i int takes - %zu microseconds\n", n, time); - - gettimeofday(&start, NULL); - for(i=0; i < n; i++) - lst->remove(lst, lst->count(lst) - 1); - gettimeofday(&end, NULL); - time = diff_usec(start, end); - printf("Remove from last position of %i int takes - %zu microseconds\n", n, time); - - gettimeofday(&start, NULL); - for(i=0; i < n; i++) - lst->insert(lst, &i, lst->count(lst)); - gettimeofday(&end, NULL); - time = diff_usec(start, end); - printf("Insert to last position of %i int takes - %zu microseconds\n", n, time); - - gettimeofday(&start, NULL); - for(i=0; i < n; i++) - lst->replace(lst, &n, i); - gettimeofday(&end, NULL); - time = diff_usec(start, end); - printf("Replace of %i int takes - %zu microseconds\n", n, time); - - printf("\n"); - lst->free(lst); - - return 0; - -} /**** MAIN ****/ - -#if defined(WIN32) -int gettimeofday(struct timeval *tp, void *tzp) -{ - LARGE_INTEGER ticksPerSecond; - LARGE_INTEGER tick; - LARGE_INTEGER time; - - if (QueryPerformanceFrequency(&ticksPerSecond) == 0) - return -1; - QueryPerformanceCounter(&tick); /* what time is it? */ - - time.QuadPart = tick.QuadPart / ticksPerSecond.QuadPart; /* convert the tick number into the number */ - /* of seconds since the system was started... */ - tp->tv_sec = time.QuadPart; /* seconds */ - tp->tv_usec = (tick.QuadPart - (time.QuadPart * ticksPerSecond.QuadPart)) * - 1000000.0 / ticksPerSecond.QuadPart; - return 0; -} -#endif diff --git a/content/chapters/compute/lab/support/create-process/Makefile b/content/chapters/compute/lab/support/create-process/Makefile deleted file mode 100644 index 4c26677077..0000000000 --- a/content/chapters/compute/lab/support/create-process/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARY = fork -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/compute/lab/support/fork-faults/Makefile b/content/chapters/compute/lab/support/fork-faults/Makefile deleted file mode 100644 index 090cfc26fc..0000000000 --- a/content/chapters/compute/lab/support/fork-faults/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARY = fork_faults -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/compute/lab/support/libult/Makefile b/content/chapters/compute/lab/support/libult/Makefile deleted file mode 100644 index 3c1136ddeb..0000000000 --- a/content/chapters/compute/lab/support/libult/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -TEST = test_ult -LIBULT = libult.so - -all: $(TEST) $(LIBULT) - -include ../../../../../common/makefile/linux.mk - -LIB_OBJECTS = queue.o tcb.o threads.o -TEST_OBJECTS = test_ult.o $(LOGGER) -LDFLAGS = -L. -LDLIBS = -lult - -$(TEST): $(TEST_OBJECTS) $(LIBULT) - $(CC) -o $@ $(TEST_OBJECTS) $(LDFLAGS) $(LDLIBS) - -$(LIBULT): $(LIB_OBJECTS) - $(CC) -shared -o $@ $^ - -clean:: - rm -f $(TEST) $(LIBULT) - -.PHONY: all clean diff --git a/content/chapters/compute/lab/support/libult/queue.h b/content/chapters/compute/lab/support/libult/queue.h deleted file mode 100644 index c7e4f14dda..0000000000 --- a/content/chapters/compute/lab/support/libult/queue.h +++ /dev/null @@ -1,74 +0,0 @@ -/** -* MIT License -* -* Copyright (c) 2020 Andreas Schärtl and Contributors -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -/* Github link: https://github.com/kissen/threads */ - -/* - * queue.h - * Defines a queue to mange TCB elements. - */ - -#ifndef QUEUE_H -#define QUEUE_H - - -#include "tcb.h" - -#include - - -typedef struct queue QUEUE; - - -/* Create a new initialized QUEUE on the heap. Returns a pointer to - the new block or NULL on error. */ -QUEUE *queue_new(void); - - -/* Destroy queue, freeing all associated memory with it. It also frees - all memory of the elements inside the queue. */ -void queue_destroy(QUEUE *queue); - - -/* Return the number of items in queue. */ -size_t queue_size(const QUEUE *queue); - - -/* Add elem to the end of queue. Returns 0 on succes and non-zero on - failure.*/ -int queue_enqueue(QUEUE *queue, TCB *elem); - - -/* Remove the first item from the queue and return it. The caller will - have to free the reuturned element. Returns NULL if the queue is - empty. */ -TCB *queue_dequeue(QUEUE *queue); - - -/* Remove element with matching id from the queue. Returns the removed - element or NULL if no such element exists. */ -TCB *queue_remove_id(QUEUE *queue, int id); - - -#endif diff --git a/content/chapters/compute/lab/support/libult/tcb.c b/content/chapters/compute/lab/support/libult/tcb.c deleted file mode 100644 index 383fb4bf97..0000000000 --- a/content/chapters/compute/lab/support/libult/tcb.c +++ /dev/null @@ -1,54 +0,0 @@ -/** -* MIT License -* -* Copyright (c) 2020 Andreas Schärtl and Contributors -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -/* Github link: https://github.com/kissen/threads */ - -#include "tcb.h" - -#include - - -TCB *tcb_new(void) -{ - static int next_id = 1; - - TCB *new; - - if ((new = calloc(1, sizeof(TCB))) == NULL) { - return NULL; - } - - new->id = next_id++; - return new; -} - - -void tcb_destroy(TCB *block) -{ - if (block->has_dynamic_stack) { - free(block->context.uc_stack.ss_sp); - } - - free(block); -} diff --git a/content/chapters/compute/lab/support/libult/tcb.h b/content/chapters/compute/lab/support/libult/tcb.h deleted file mode 100644 index ab95460aee..0000000000 --- a/content/chapters/compute/lab/support/libult/tcb.h +++ /dev/null @@ -1,61 +0,0 @@ -/** -* MIT License -* -* Copyright (c) 2020 Andreas Schärtl and Contributors -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -/* Github link: https://github.com/kissen/threads */ - -/* - * tcb.h - * Defines the TCB (thread control block) data structure and functions - * to work with it. - */ - -#ifndef TCB_H -#define TCB_H - - -#include -#include -#include - - -typedef struct { - int id; - ucontext_t context; - bool has_dynamic_stack; - void *(*start_routine) (void *); - void *argument; - void *return_value; -} TCB; - - -/* Create a new zeroed TCB on the heap. Returns a pointer to the new - block or NULL on error. */ -TCB *tcb_new(void); - - -/* Destroy block, freeing all associated memory with it */ -void tcb_destroy(TCB *block); - - -#endif diff --git a/content/chapters/compute/lab/support/libult/threads.h b/content/chapters/compute/lab/support/libult/threads.h deleted file mode 100644 index 4c17c1e5c4..0000000000 --- a/content/chapters/compute/lab/support/libult/threads.h +++ /dev/null @@ -1,55 +0,0 @@ -/** -* MIT License -* -* Copyright (c) 2020 Andreas Schärtl and Contributors -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -/* Github link: https://github.com/kissen/threads */ - -/* - * threads.h - * Interface to a barebones user level thread library. - */ - -#ifndef THREADS_H -#define THREADS_H - - -/* Create a new thread. func is the function that will be run once the - thread starts execution and arg is the argument for that - function. On success, returns an id equal or greater to 0. On - failure, errno is set and -1 is returned. */ -int threads_create(void *(*start_routine) (void *), void *arg); - - -/* Stop execution of the thread calling this function. */ -void threads_exit(void *result); - - -/* Wait for the thread with matching id to finish execution, that is, - for it to call threads_exit. On success, the threads result is - written into result and id is returned. If no completed thread with - matching id exists, 0 is returned. On error, -1 is returned and - errno is set. */ -int threads_join(int id, void **result); - - -#endif diff --git a/content/chapters/compute/lab/support/mini-shell/.gitignore b/content/chapters/compute/lab/support/mini-shell/.gitignore deleted file mode 100644 index fb647f35e4..0000000000 --- a/content/chapters/compute/lab/support/mini-shell/.gitignore +++ /dev/null @@ -1 +0,0 @@ -mini_shell diff --git a/content/chapters/compute/lab/support/mini-shell/Makefile b/content/chapters/compute/lab/support/mini-shell/Makefile deleted file mode 100644 index 0c9ef220df..0000000000 --- a/content/chapters/compute/lab/support/mini-shell/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARY = mini_shell -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/compute/lab/support/mini-shell/mini_shell.c b/content/chapters/compute/lab/support/mini-shell/mini_shell.c deleted file mode 100644 index 99a511032e..0000000000 --- a/content/chapters/compute/lab/support/mini-shell/mini_shell.c +++ /dev/null @@ -1,117 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define MAX_LINE_SIZE 256 -#define MAX_ARGS 8 - -#define ERROR 0 -#define SIMPLE 1 -#define REDIRECT 2 -#define PIPE 3 -#define EXIT_CMD 5 - -static char *verb; -static char **args; -static char *stdin_file; -static char *stdout_file; -static char *stderr_file; - -static int type; - - -static void alloc_mem(void) -{ - args = malloc(MAX_ARGS * sizeof(char *)); -} - -static int parse_line(char *line) -{ - int ret = SIMPLE; - int idx = 0; - char *token; - char *delim = "=\n";; - char *saveptr = NULL; - - stdin_file = NULL; - stdout_file = NULL; - stderr_file = NULL; - - /* Check for exit. */ - if (strncmp("exit", line, strlen("exit")) == 0) - return EXIT_CMD; - - /* Normal command. */ - delim = " \t\n"; - token = strtok_r(line, delim, &saveptr); - - if (token == NULL) - return ERROR; - - verb = strdup(token); - - /* Copy args. */ - idx = 0; - while (token != NULL) { - if (token == NULL) { - printf(" Expansion failed\n"); - return ERROR; - } - - args[idx++] = strdup(token); - token = strtok_r(NULL, delim, &saveptr); - } - - args[idx++] = NULL; - return ret; -} - -/** - * @args - array that contains a simple command with parameters - */ -static void simple_cmd(char **args) -{ - (void)args; - /** - * TODO - Create a process to execute the command. - * Use `execvp` to launch the new process. - */ -} - -int main(void) -{ - char line[MAX_LINE_SIZE]; - - alloc_mem(); - - while (1) { - printf("> "); - fflush(stdout); - - memset(line, 0, MAX_LINE_SIZE); - - if (fgets(line, sizeof(line), stdin) == NULL) - exit(EXIT_SUCCESS); - - type = parse_line(line); - - switch (type) { - case EXIT_CMD: - exit(EXIT_SUCCESS); - - case SIMPLE: - simple_cmd(args); - break; - } - } - - return 0; -} diff --git a/content/chapters/compute/lab/support/multithreaded/.gitignore b/content/chapters/compute/lab/support/multithreaded/.gitignore deleted file mode 100644 index 6da400737e..0000000000 --- a/content/chapters/compute/lab/support/multithreaded/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/multithreaded diff --git a/content/chapters/compute/lab/support/multithreaded/Makefile b/content/chapters/compute/lab/support/multithreaded/Makefile deleted file mode 100644 index a7c5b596ec..0000000000 --- a/content/chapters/compute/lab/support/multithreaded/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -BINARY = multithreaded -LDLIBS = -lpthread -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/compute/lab/support/multithreaded/multithreaded.c b/content/chapters/compute/lab/support/multithreaded/multithreaded.c deleted file mode 100644 index f0f555a4e6..0000000000 --- a/content/chapters/compute/lab/support/multithreaded/multithreaded.c +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define __unused __attribute__((unused)) - -static void wait_for_input(const char *msg) -{ - char buffer[128]; - - printf("%s ...", msg); - fgets(buffer, 128, stdin); -} - -static void *sleep_wrapper(void __unused *data) -{ - sleep(100); - return NULL; -} - -#define NUM_THREADS 5 - -int main(void) -{ - size_t i; - pthread_t tid[NUM_THREADS]; - int rc; - - wait_for_input("Press key to start creating threads"); - - for (i = 0; i < NUM_THREADS; i++) { - rc = pthread_create(&tid[i], NULL, sleep_wrapper, NULL); - DIE(rc < 0, "pthread_create"); - wait_for_input("Press key to create new thread"); - } - - wait_for_input("Press key to wait for threads"); - for (i = 0; i < NUM_THREADS; i++) { - pthread_join(tid[i], NULL); - } - - return 0; -} diff --git a/content/chapters/compute/lab/support/page-faults/Makefile b/content/chapters/compute/lab/support/page-faults/Makefile deleted file mode 100644 index 816aaffe10..0000000000 --- a/content/chapters/compute/lab/support/page-faults/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -BINARIES = page_faults file.txt -include ../../../../../common/makefile/multiple.mk - -file.txt: - curl metaphorpsum.com/paragraphs/20/50 > $@ diff --git a/content/chapters/compute/lab/support/race-condition/c/Makefile b/content/chapters/compute/lab/support/race-condition/c/Makefile deleted file mode 100644 index 76064f6a01..0000000000 --- a/content/chapters/compute/lab/support/race-condition/c/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -BINARIES = race_condition race_condition_tls race_condition_mutex race_condition_atomic race_condition_atomic2 -LDLIBS = -lpthread -include ../../../../../../common/makefile/multiple.mk diff --git a/content/chapters/compute/lab/support/shared-memory/Makefile b/content/chapters/compute/lab/support/shared-memory/Makefile deleted file mode 100644 index fb6f13a0db..0000000000 --- a/content/chapters/compute/lab/support/shared-memory/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -LDLIBS = -lpthread -BINARY = shared_memory -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/compute/lab/support/sleepy/Makefile b/content/chapters/compute/lab/support/sleepy/Makefile deleted file mode 100644 index c35f3e9e4e..0000000000 --- a/content/chapters/compute/lab/support/sleepy/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -BINARY = sleepy_creator -include ../../../../../common/makefile/single.mk - -clean:: - -rm -f syscalls* diff --git a/content/chapters/compute/lab/support/sum-array-bugs/seg-fault/Makefile b/content/chapters/compute/lab/support/sum-array-bugs/seg-fault/Makefile deleted file mode 100644 index b0c9c9ad2d..0000000000 --- a/content/chapters/compute/lab/support/sum-array-bugs/seg-fault/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -BINARIES = sum_array_threads sum_array_processes -include ../../../../../../common/makefile/multiple.mk -LDLIBS += -lpthread - -sum_array_threads: sum_array_threads.o generate_random_array.o - -sum_array_processes: sum_array_processes.o generate_random_array.o diff --git a/content/chapters/compute/lab/support/sum-array/c/Makefile b/content/chapters/compute/lab/support/sum-array/c/Makefile deleted file mode 100644 index 71cbcc95b6..0000000000 --- a/content/chapters/compute/lab/support/sum-array/c/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -BINARIES = sum_array_sequential sum_array_threads sum_array_processes sum_array_threads_openmp -CFLAGS += -fopenmp -LDFLAGS += -fopenmp -include ../../../../../../common/makefile/multiple.mk - -sum_array_sequential: sum_array_sequential.o generate_random_array.o - -sum_array_threads: sum_array_threads.o generate_random_array.o - -sum_array_processes: sum_array_processes.o generate_random_array.o - -sum_array_threads_openmp: sum_array_threads_openmp.o generate_random_array.o diff --git a/content/chapters/compute/lab/support/wait-for-me/Makefile b/content/chapters/compute/lab/support/wait-for-me/Makefile deleted file mode 100644 index 74a6cd58eb..0000000000 --- a/content/chapters/compute/lab/support/wait-for-me/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARY = wait_for_me_threads -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/compute/lecture/Makefile b/content/chapters/compute/lecture/Makefile deleted file mode 100644 index 3a42bc8eaf..0000000000 --- a/content/chapters/compute/lecture/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -TARGETS = round-robin race-condition race-condition-toctou race-condition-lock -include ../../../common/makefile/slides.mk diff --git a/content/chapters/compute/lecture/media/copy-on-write-final.svg b/content/chapters/compute/lecture/media/copy-on-write-final.svg deleted file mode 100644 index 33e8245fa5..0000000000 --- a/content/chapters/compute/lecture/media/copy-on-write-final.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Frame A
Frame A
Frame B
Frame B
Frame C
Frame C
Physical Memory
Physical Memory
Code Page
Code Page
.rodata Page
.rodata Page
Heap Page
Heap Page
After
Memory
Write
After...
Parent Process
Page Table
Parent Process...
Child Process Page Table
Child Process...
Code Page
Code Page
.rodata Page
.rodata Page
Heap Page
Heap Page
r--
r--
rw-
rw-
r-x
r-x
Copy of Frame C
Copy of Frame C Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/compute/lecture/media/copy-on-write-initial.svg b/content/chapters/compute/lecture/media/copy-on-write-initial.svg deleted file mode 100644 index 0387aa5ed9..0000000000 --- a/content/chapters/compute/lecture/media/copy-on-write-initial.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Frame B
Frame B
Frame C
Frame C
Physical Memory
Physical Memory
.rodata Page
.rodata Page
Heap Page
Heap Page
Parent Process Page Table
Parent Process...
Child Process Page Table
Child Process...
Heap Page
Heap Page
fork()
fork()
r--
r--
rw-
rw-
r-x
r-x
Code Page
Code Page
Frame A
Frame A
Code Page
Code Page
.rodata Page
.rodata PageText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/compute/lecture/slides.mdpp b/content/chapters/compute/lecture/slides.mdpp deleted file mode 100644 index 646b07e9d9..0000000000 --- a/content/chapters/compute/lecture/slides.mdpp +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: "OS: Compute" -revealOptions: - background-color: 'aquamarine' - transition: 'none' - slideNumber: true - autoAnimateDuration: 0.0 ---- - -# COMPUTE - -1. [Processes](#1-processes) -1. [Threads](#2-threads) -1. [Scheduling](#3-scheduling) -1. [Synchronization](#4-synchronization) - - -!INCLUDE "slides/intro.md" - -!INCLUDE "slides/processes.md" - -!INCLUDE "slides/threads.md" - -!INCLUDE "slides/process-attributes.md" - -!INCLUDE "slides/fork.md" - -!INCLUDE "slides/copy-on-write.md" - -!INCLUDE "slides/scheduling.md" - -!INCLUDE "slides/scheduling-algorithms.md" - -!INCLUDE "slides/synchronization.md" - -!INCLUDE "slides/mutual-exclusion.md" - -!INCLUDE "slides/notifications.md" - -!INCLUDE "slides/barrier.md" - -!INCLUDE "slides/thread-safety.md" - -!INCLUDE "slides/cool-extra-stuff.md" diff --git a/content/chapters/io/lab/solution/buffering/diy_buffering.c b/content/chapters/io/lab/solution/buffering/diy_buffering.c index 94026076bf..c1f42274f9 100644 --- a/content/chapters/io/lab/solution/buffering/diy_buffering.c +++ b/content/chapters/io/lab/solution/buffering/diy_buffering.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/solution/file-mappings/mmap_cp.c b/content/chapters/io/lab/solution/file-mappings/mmap_cp.c index 8805ad3069..263aca9e87 100644 --- a/content/chapters/io/lab/solution/file-mappings/mmap_cp.c +++ b/content/chapters/io/lab/solution/file-mappings/mmap_cp.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/solution/mini-shell/Makefile b/content/chapters/io/lab/solution/mini-shell/Makefile deleted file mode 120000 index 2a024fd2c8..0000000000 --- a/content/chapters/io/lab/solution/mini-shell/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../../../compute/lab/support/mini-shell/Makefile \ No newline at end of file diff --git a/content/chapters/io/lab/solution/mini-shell/Makefile b/content/chapters/io/lab/solution/mini-shell/Makefile new file mode 100644 index 0000000000..ae810113b5 --- /dev/null +++ b/content/chapters/io/lab/solution/mini-shell/Makefile @@ -0,0 +1,41 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +SRCS = mini_shell.c +OBJS = $(SRCS:.c=.o) +BINARIES = mini_shell + +# Default rule: Build everything +all: $(BINARIES) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile object files from source files +%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create mini_shell binary +mini_shell: mini_shell.o $(LOGGER) + $(CC) $(CFLAGS) mini_shell.o $(LOGGER) -o mini_shell $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean \ No newline at end of file diff --git a/content/chapters/io/lab/solution/mini-shell/utils/log/CPPLINT.cfg b/content/chapters/io/lab/solution/mini-shell/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/content/chapters/io/lab/solution/mini-shell/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/content/chapters/io/lab/solution/mini-shell/utils/log/log.c b/content/chapters/io/lab/solution/mini-shell/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/content/chapters/io/lab/solution/mini-shell/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/content/chapters/io/lab/solution/mini-shell/utils/log/log.h b/content/chapters/io/lab/solution/mini-shell/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/content/chapters/io/lab/solution/mini-shell/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.c b/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.h b/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/content/chapters/io/lab/solution/mini-shell/utils/utils.h b/content/chapters/io/lab/solution/mini-shell/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/content/chapters/io/lab/solution/mini-shell/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/io/lab/solution/receive-challenges/receive_fifo.c b/content/chapters/io/lab/solution/receive-challenges/receive_fifo.c index 140386a267..0088000554 100644 --- a/content/chapters/io/lab/solution/receive-challenges/receive_fifo.c +++ b/content/chapters/io/lab/solution/receive-challenges/receive_fifo.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/solution/sparse-file/solve.c b/content/chapters/io/lab/solution/sparse-file/solve.c index aa40d232b7..b7eff24c4e 100644 --- a/content/chapters/io/lab/solution/sparse-file/solve.c +++ b/content/chapters/io/lab/solution/sparse-file/solve.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/support/buffering/diy_buffering.c b/content/chapters/io/lab/support/buffering/diy_buffering.c index ce0a3cdd98..dbb1258c87 100644 --- a/content/chapters/io/lab/support/buffering/diy_buffering.c +++ b/content/chapters/io/lab/support/buffering/diy_buffering.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/support/buffering/libc_buffering.c b/content/chapters/io/lab/support/buffering/libc_buffering.c index 4d75651fa7..03d8cfbd96 100644 --- a/content/chapters/io/lab/support/buffering/libc_buffering.c +++ b/content/chapters/io/lab/support/buffering/libc_buffering.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/support/buffering/no_buffering.c b/content/chapters/io/lab/support/buffering/no_buffering.c index af4ec04eb2..b9bb9c21de 100644 --- a/content/chapters/io/lab/support/buffering/no_buffering.c +++ b/content/chapters/io/lab/support/buffering/no_buffering.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/support/buffering/printf_buffering.c b/content/chapters/io/lab/support/buffering/printf_buffering.c index 6db27d2ae4..c21acfb973 100644 --- a/content/chapters/io/lab/support/buffering/printf_buffering.c +++ b/content/chapters/io/lab/support/buffering/printf_buffering.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include diff --git a/content/chapters/io/lab/support/file-mappings/mmap_cp.c b/content/chapters/io/lab/support/file-mappings/mmap_cp.c index e5a1a5f9af..cdf83d33c6 100644 --- a/content/chapters/io/lab/support/file-mappings/mmap_cp.c +++ b/content/chapters/io/lab/support/file-mappings/mmap_cp.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/support/mini-shell/Makefile b/content/chapters/io/lab/support/mini-shell/Makefile deleted file mode 120000 index 2a024fd2c8..0000000000 --- a/content/chapters/io/lab/support/mini-shell/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../../../compute/lab/support/mini-shell/Makefile \ No newline at end of file diff --git a/content/chapters/io/lab/support/mini-shell/Makefile b/content/chapters/io/lab/support/mini-shell/Makefile new file mode 100644 index 0000000000..ae810113b5 --- /dev/null +++ b/content/chapters/io/lab/support/mini-shell/Makefile @@ -0,0 +1,41 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +SRCS = mini_shell.c +OBJS = $(SRCS:.c=.o) +BINARIES = mini_shell + +# Default rule: Build everything +all: $(BINARIES) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile object files from source files +%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create mini_shell binary +mini_shell: mini_shell.o $(LOGGER) + $(CC) $(CFLAGS) mini_shell.o $(LOGGER) -o mini_shell $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean \ No newline at end of file diff --git a/content/chapters/io/lab/support/mini-shell/utils/log/CPPLINT.cfg b/content/chapters/io/lab/support/mini-shell/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/content/chapters/io/lab/support/mini-shell/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/content/chapters/io/lab/support/mini-shell/utils/log/log.c b/content/chapters/io/lab/support/mini-shell/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/content/chapters/io/lab/support/mini-shell/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/content/chapters/io/lab/support/mini-shell/utils/log/log.h b/content/chapters/io/lab/support/mini-shell/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/content/chapters/io/lab/support/mini-shell/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.c b/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.c new file mode 100644 index 0000000000..78581b2239 --- /dev/null +++ b/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.h b/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.h new file mode 100644 index 0000000000..906976dc94 --- /dev/null +++ b/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/* + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/content/chapters/io/lab/support/mini-shell/utils/utils.h b/content/chapters/io/lab/support/mini-shell/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/content/chapters/io/lab/support/mini-shell/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/io/lab/support/receive-challenges/receive_fifo.c b/content/chapters/io/lab/support/receive-challenges/receive_fifo.c index 52dfcd4bce..ba06bdc9ca 100644 --- a/content/chapters/io/lab/support/receive-challenges/receive_fifo.c +++ b/content/chapters/io/lab/support/receive-challenges/receive_fifo.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/support/redirect/redirect_parallel.c b/content/chapters/io/lab/support/redirect/redirect_parallel.c index 1f81b41ca1..32bfe8a820 100644 --- a/content/chapters/io/lab/support/redirect/redirect_parallel.c +++ b/content/chapters/io/lab/support/redirect/redirect_parallel.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/support/sparse-file/generate.c b/content/chapters/io/lab/support/sparse-file/generate.c index 3708eecfa1..f0494021a2 100644 --- a/content/chapters/io/lab/support/sparse-file/generate.c +++ b/content/chapters/io/lab/support/sparse-file/generate.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/support/sparse-file/solve.c b/content/chapters/io/lab/support/sparse-file/solve.c index 3f7859c77c..a7e9df934d 100644 --- a/content/chapters/io/lab/support/sparse-file/solve.c +++ b/content/chapters/io/lab/support/sparse-file/solve.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lecture/demo/devices/filesystem-size.c b/content/chapters/io/lecture/demo/devices/filesystem-size.c index 66eb95a67c..cc98d5f496 100644 --- a/content/chapters/io/lecture/demo/devices/filesystem-size.c +++ b/content/chapters/io/lecture/demo/devices/filesystem-size.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause /* * Use ioctl to get number of bytes in filesystem. diff --git a/content/chapters/io/lecture/demo/devices/hwaddr-ioctl.c b/content/chapters/io/lecture/demo/devices/hwaddr-ioctl.c index 5cd0d5474a..ae43d67720 100644 --- a/content/chapters/io/lecture/demo/devices/hwaddr-ioctl.c +++ b/content/chapters/io/lecture/demo/devices/hwaddr-ioctl.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause /* * Use ioctl to get MAC address of network interface. diff --git a/content/chapters/io/lecture/demo/file-interface/c-file-ops.c b/content/chapters/io/lecture/demo/file-interface/c-file-ops.c index 04a03c29ef..04dc9a1ff4 100644 --- a/content/chapters/io/lecture/demo/file-interface/c-file-ops.c +++ b/content/chapters/io/lecture/demo/file-interface/c-file-ops.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lecture/demo/file-interface/close-stdout.c b/content/chapters/io/lecture/demo/file-interface/close-stdout.c index 32ee749626..1d358fa407 100644 --- a/content/chapters/io/lecture/demo/file-interface/close-stdout.c +++ b/content/chapters/io/lecture/demo/file-interface/close-stdout.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lecture/demo/file-interface/open-vs-dup.c b/content/chapters/io/lecture/demo/file-interface/open-vs-dup.c index a81bba76a6..58d4a3daf5 100644 --- a/content/chapters/io/lecture/demo/file-interface/open-vs-dup.c +++ b/content/chapters/io/lecture/demo/file-interface/open-vs-dup.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lecture/demo/file-interface/sparse-file.c b/content/chapters/io/lecture/demo/file-interface/sparse-file.c index 58cc4f966b..52e380a06d 100644 --- a/content/chapters/io/lecture/demo/file-interface/sparse-file.c +++ b/content/chapters/io/lecture/demo/file-interface/sparse-file.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lecture/demo/file-interface/truncate.c b/content/chapters/io/lecture/demo/file-interface/truncate.c index 1f668bf277..460cc9b806 100644 --- a/content/chapters/io/lecture/demo/file-interface/truncate.c +++ b/content/chapters/io/lecture/demo/file-interface/truncate.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lecture/demo/optimizations/fwrite-10000.c b/content/chapters/io/lecture/demo/optimizations/fwrite-10000.c index 3e45f4abcc..fe1ee14600 100644 --- a/content/chapters/io/lecture/demo/optimizations/fwrite-10000.c +++ b/content/chapters/io/lecture/demo/optimizations/fwrite-10000.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lecture/demo/optimizations/stdout-vs-stderr.c b/content/chapters/io/lecture/demo/optimizations/stdout-vs-stderr.c index e22ffd3597..fe56a4cc09 100644 --- a/content/chapters/io/lecture/demo/optimizations/stdout-vs-stderr.c +++ b/content/chapters/io/lecture/demo/optimizations/stdout-vs-stderr.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lecture/demo/optimizations/write-1000-unbuf.c b/content/chapters/io/lecture/demo/optimizations/write-1000-unbuf.c index 5740266f68..6a16751851 100644 --- a/content/chapters/io/lecture/demo/optimizations/write-1000-unbuf.c +++ b/content/chapters/io/lecture/demo/optimizations/write-1000-unbuf.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lecture/demo/optimizations/write-10000.c b/content/chapters/io/lecture/demo/optimizations/write-10000.c index 81a3c91989..62f0bcba4f 100644 --- a/content/chapters/io/lecture/demo/optimizations/write-10000.c +++ b/content/chapters/io/lecture/demo/optimizations/write-10000.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/common/test/c/open.c b/content/common/test/c/open.c index 2f78213d96..23bbf5f131 100644 --- a/content/common/test/c/open.c +++ b/content/common/test/c/open.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +// SPDX-License-Identifier: BSD-3-Clause /* * Test case for common files: Makefile, logging, DIE, ERR macros. */ diff --git a/gen-view.py b/gen-view.py index f70addcd9f..151925593f 100644 --- a/gen-view.py +++ b/gen-view.py @@ -240,7 +240,9 @@ def get_file_to_lab_dict(self) -> dict: self.fileToLab = {} for id, lab in enumerate(self.data["lab_structure"]): - for c in lab["content"]: + tasks = lab["tasks"] if "tasks" in lab else [] + guides = lab["guides"] if "guides" in lab else [] + for c in lab["content"] + tasks + guides: self.fileToLab[c] = f"lab{id+1}.md" return self.fileToLab