From 8195788ef6571c0dc44e93920f9911c3229f954c Mon Sep 17 00:00:00 2001 From: Moros Smith Date: Wed, 8 May 2024 15:17:08 -0400 Subject: [PATCH 1/3] catch compiler and linker command timeout --- app/Http/Controllers/CodeController.php | 53 ++++++++++++++++++------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/app/Http/Controllers/CodeController.php b/app/Http/Controllers/CodeController.php index efe258a..88ef0d6 100644 --- a/app/Http/Controllers/CodeController.php +++ b/app/Http/Controllers/CodeController.php @@ -512,6 +512,7 @@ function compileCode($code) "pgetinker.cpp", "-o", "pgetinker.o", + "-std=c++20", ]); $log->info("preparing linker command"); @@ -531,19 +532,31 @@ function compileCode($code) "-sUSE_SDL_MIXER=2", "-sLLD_REPORT_UNDEFINED", "-sSINGLE_FILE", + "-std=c++20", ]); - $log->info("invoking the compiler"); - $compilerProcessResult = Process::env($environmentVariables) - ->path($workspaceDirectory) - ->timeout(10) - ->command($compilerCommand)->run(); - - $log->info("compiler exited with code: " . $compilerProcessResult->exitCode()); - $response = null; - if($compilerProcessResult->exitCode() !== 0) + $log->info("invoking the compiler"); + try + { + $compilerProcessResult = Process::env($environmentVariables) + ->path($workspaceDirectory) + ->timeout(10) + ->command($compilerCommand)->run(); + + $log->info("compiler exited with code: " . $compilerProcessResult->exitCode()); + } + catch(Exception $e) + { + $response = [ + "statusCode" => 400, + "stdout" => "", + "stderr" => "compiler timed out. your code is either broken or there's too much of it!", + ]; + } + + if($response == null) { $response = [ "statusCode" => 400, @@ -561,11 +574,23 @@ function compileCode($code) if($response == null) { $log->info("invoking the linker"); - $linkerProcessResult = Process::env($environmentVariables) - ->path($workspaceDirectory) - ->timeout(10) - ->command($linkerCommand)->run(); - + + try + { + $linkerProcessResult = Process::env($environmentVariables) + ->path($workspaceDirectory) + ->timeout(10) + ->command($linkerCommand)->run(); + } + catch(Exception $e) + { + $response = [ + "statusCode" => 400, + "stdout" => "", + "stderr" => "linker timed out. your code is either broken or there's too much of it or there's some other bug, report it?", + ]; + } + if($linkerProcessResult->exitCode() !== 0) { $response = [ From 01d3a883ca9f6601746ac89a8c9354331a53929d Mon Sep 17 00:00:00 2001 From: Moros Smith Date: Wed, 8 May 2024 15:19:50 -0400 Subject: [PATCH 2/3] update changelog add Utils, move hashCode from CodeController to Utils update ignored files compiler test sources work-in-progress compiler tests update files docker should ignore use cgroups work-in-progress compiler class process remote includes prepare environment update example environment finish compiler refactor compiler tests refactor the controller --- .dockerignore | 3 +- .env.example | 7 +- .gitignore | 1 + CHANGELOG.md | 1 + Dockerfile | 1 + app/Http/Controllers/CodeController.php | 571 ++---------------- composer.json | 6 +- docker/docker-entrypoint.sh | 4 + pgetinker/Compiler.php | 550 +++++++++++++++++ pgetinker/Utils.php | 75 +++ tests/Feature/CodeControllerHashTest.php | 56 -- tests/Feature/CompilerTest.php | 89 ++- tests/Feature/UtilsHashCodeTest.php | 49 ++ .../absolute-or-relative.cpp | 11 + .../Feature/compiler-test-source/example.cpp | 76 +++ .../compiler-test-source/hello-world.cpp | 7 + .../compiler-test-source/ice-timeout.cpp | 6 + .../compiler-test-source/remote-includes.cpp | 1 + third_party/nsjail-emscripten.cfg | 20 + 19 files changed, 931 insertions(+), 603 deletions(-) create mode 100644 pgetinker/Compiler.php create mode 100644 pgetinker/Utils.php delete mode 100644 tests/Feature/CodeControllerHashTest.php create mode 100644 tests/Feature/UtilsHashCodeTest.php create mode 100644 tests/Feature/compiler-test-source/absolute-or-relative.cpp create mode 100644 tests/Feature/compiler-test-source/example.cpp create mode 100644 tests/Feature/compiler-test-source/hello-world.cpp create mode 100644 tests/Feature/compiler-test-source/ice-timeout.cpp create mode 100644 tests/Feature/compiler-test-source/remote-includes.cpp diff --git a/.dockerignore b/.dockerignore index 8bf03e4..9d9f38f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,4 +6,5 @@ vendor docker-compose.yml storage/app/workspaces storage/app/compilerCache -storage/logs/laravel.log \ No newline at end of file +storage/logs/laravel.log +tests/Feature/test_* diff --git a/.env.example b/.env.example index 1baa00f..3b360dc 100644 --- a/.env.example +++ b/.env.example @@ -12,10 +12,11 @@ APP_FAKER_LOCALE=en_US APP_MAINTENANCE_DRIVER=file APP_MAINTENANCE_STORE=database -COMPILER_ENVIRONMENT=nsjail COMPILER_CACHING=true -REMOTE_INCLUDE_CACHING=true -PROCESSING_TIMEOUT=5 +COMPILER_ENVIRONMENT=nsjail +COMPILER_TIMEOUT=10 +COMPILER_CODE_PROCESSING_TIMEOUT=5 +COMPILER_REMOTE_INCLUDE_CACHING=true BCRYPT_ROUNDS=12 diff --git a/.gitignore b/.gitignore index 8bc6884..3ce55d3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ yarn-error.log third_party/v*/lib third_party/v*/model.h +tests/Feature/test_* diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e9aa52..35f6280 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ It is a summary of changes that would be pertinent to the end user of the PGEtin ## 2024-05-08 - Added limitations to the remote include feature +- Catch compiler/linker timeout for better error handling ## 2024-05-06 diff --git a/Dockerfile b/Dockerfile index d9a4615..c8f9f5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,7 @@ RUN apt-get -y update && \ libtool \ make \ pkg-config \ + cgroup-tools \ protobuf-compiler && \ rm -rf /var/lib/apt/lists/* diff --git a/app/Http/Controllers/CodeController.php b/app/Http/Controllers/CodeController.php index 88ef0d6..d9f1c24 100644 --- a/app/Http/Controllers/CodeController.php +++ b/app/Http/Controllers/CodeController.php @@ -4,16 +4,14 @@ use App\Models\Code; use Exception; -use Illuminate\Http\Client\PendingRequest; use Illuminate\Http\Request; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Process; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; -use Monolog\Formatter\LineFormatter; -use Monolog\Handler\StreamHandler; -use Monolog\Logger; -use Illuminate\Support\Facades\Http; +use PGEtinker\Compiler; + +use function PGEtinker\Utils\hashCode; class CodeController extends Controller { @@ -21,7 +19,6 @@ function Compile(Request $request) { $result = $this->compileCode($request->input("code", null)); unset($result["hash"]); - return response($result, $result["statusCode"])->header("Content-Type", "application/json"); } @@ -81,76 +78,6 @@ function Share(Request $request) return response([ "statusCode" => 500, "message" => "some major server malfunction" ], 500)->header("Content-Type", "application/json"); } - function hashCode($code) - { - /** - * preemptively pad each of these tokens with tokens - */ - foreach([ - "(","[","{",")","]","}",",","-","+","*","=" - ] as $token) - { - $code = str_replace($token, " {$token} ", $code); - } - - /** - * thanks Bixxy for the general idea here. - */ - $tokens = token_get_all(" 1) - $text = " "; - - } - - // if, for any reason we reach here, add it to the code we wanna hash - $cppcode .= $text; - continue; - } - else - { - // any other token is passed through, as is. - $cppcode .= $token; - } - } - - // take off multiple new lines left over - $cppcode = preg_replace('/\n\s*\n/', "\n", $cppcode); - - return hash("sha256", $cppcode); - } function compileCode($code) { @@ -173,33 +100,25 @@ function compileCode($code) ]; } - $localDisk = Storage::disk("local"); - $remoteDisk = (!empty(env("AWS_BUCKET"))) ? Storage::disk("s3") : Storage::disk("local"); - - $hashedCode = $this->hashCode($code); + $hashedCode = hashCode($code); if(env("COMPILER_CACHING", false)) { - if(!$remoteDisk->exists("compilerCache")) + if(Storage::directoryMissing("compilerCache")) { - $remoteDisk->makeDirectory("compilerCache"); + Storage::makeDirectory("compilerCache"); } - if(!$remoteDisk->exists("remoteIncludeCache")) + if(Storage::directoryMissing("remoteIncludeCache")) { - $remoteDisk->makeDirectory("remoteIncludeCache"); + Storage::makeDirectory("remoteIncludeCache"); } - if(!$remoteDisk->exists("workspaces")) - { - $remoteDisk->makeDirectory("workspaces"); - } - - if($remoteDisk->exists("compilerCache/{$hashedCode}")) + if(Storage::fileExists("compilerCache/{$hashedCode}")) { Log::debug("Compile: cache hit", ["hashedCode" => $hashedCode]); - $html = $remoteDisk->get("compilerCache/{$hashedCode}"); + $html = Storage::get("compilerCache/{$hashedCode}"); return [ "statusCode" => 200, @@ -211,464 +130,48 @@ function compileCode($code) Log::debug("Compile: cache miss", ["hashedCode" => $hashedCode]); } - if(!$localDisk->exists("workspaces")) + if(Storage::directoryMissing("workspaces")) { - $localDisk->makeDirectory("workspaces"); + Storage::makeDirectory("workspaces"); } - - $directoryName = "workspaces/" . Str::uuid(); - $localDisk->makeDirectory($directoryName); - - $log = new Logger("compiler"); - - $logHandler = new StreamHandler($localDisk->path($directoryName) . "/compiler.log"); - $logHandler->setFormatter(new LineFormatter(null, null, true, true)); - - $log->pushHandler($logHandler); - - Log::debug("Compile: working directory created {$directoryName}"); - - $libraryMap = [ - 'OLC_PGE_APPLICATION' => 'olcPixelGameEngine.o', - 'OLC_SOUNDWAVE_ENGINE' => 'olcSoundWaveEngine.o', - 'OLC_PGEX_GRAPHICS2D' => 'olcPGEX_Graphics2D.o', - 'OLC_PGEX_GRAPHICS3D' => 'olcPGEX_Graphics3D.o', - 'OLC_PGEX_POPUPMENU' => 'olcPGEX_PopUpMenu.o', - 'OLC_PGEX_QUICKGUI' => 'olcPGEX_QuickGUI.o', - 'OLC_PGEX_RAYCASTWORLD' => 'olcPGEX_RayCastWorld.o', - 'OLC_PGEX_SOUND' => 'olcPGEX_Sound.o', - 'OLC_PGEX_SPLASHSCREEN' => 'olcPGEX_SplashScreen.o', - 'OLC_PGEX_TRANSFORMEDVIEW' => 'olcPGEX_TransformedView.o', - 'OLC_PGEX_WIREFRAME' => 'olcPGEX_Wireframe.o', - ]; - - $linesOfCode = explode("\n", $code); - - $errors = []; - - $libraries = []; - - $log->info("begin parsing linesOfCode"); - - // mark the start of processing - $processingStartTime = microtime(true); - - // line by line code processing and filtering - for($i = 0; $i < count($linesOfCode); $i++) + if(Storage::disk("local")->exists("workspaces")) { - - $processingEndTime = microtime(true); - $processingDuration = $processingEndTime - $processingStartTime; - - if($processingDuration > floatval(env("PROCESSING_TIMEOUT", 5))) - { - $errors[] = "/pgetinker.cpp:" . $i . ":1: error: took too long to process code."; - $errors[] = " this can happen if you are using alot of remote includes"; - break; - } - - // filter include macros with an absolute or relative path, naughty naughty - preg_match( - '/^\s*#\s*i(nclude|mport)(_next)?\s+["<]((\.{1,2}|\/)[^">]*)[">]/', - $linesOfCode[$i], - $match, - PREG_OFFSET_CAPTURE, - 0 - ); - - if(count($match) > 0) - { - $errors[] = "/pgetinker.cpp:" . $i + 1 . ":1: error: absolute and relative includes are not allowed."; - $log->info("found absolute or relative path at line " . $i + 1); - continue; - } - - // filter macros to detect implementation #define - if(str_contains($linesOfCode[$i], "#define")) - { - $foundImplementationMacro = false; - foreach($libraryMap as $macro => $objectFileName) - { - if(str_contains($linesOfCode[$i], $macro)) - { - // blank the line - $linesOfCode[$i] = ""; - - // indicate that we use this library - $libraries[] = "./lib/{$objectFileName}"; - - $log->info("Found implementation macro: {$macro}"); - $foundImplementationMacro = true; - break; - } - } - - if($foundImplementationMacro) - continue; - } - - preg_match( - '/^\s*#\s*i(nclude|mport)(_next)?\s+["<](https?:\/\/(.*)[^">]*)[">]/', - $linesOfCode[$i], - $match, - PREG_OFFSET_CAPTURE, - 0 - ); - - if(count($match) > 0) - { - $log->info("found a potential url for remote include"); - - $potentialUrl = $match[3][0]; - $potentialFilename = basename($match[3][0]); - $hashedUrl = hash("sha256", $potentialUrl); - - if(env("REMOTE_INCLUDE_CACHING", false)) - { - // if we have a cached version of the url's contents, don't pull it - if( - $remoteDisk->exists("remoteIncludeCache/{$hashedUrl}") && - $remoteDisk->exists("remoteIncludeCache/{$hashedUrl}.time") - ) - { - $log->info("remote include cache hit"); - - $requestTime = floatval($remoteDisk->get("remoteIncludeCache/{$hashedUrl}.time")); - usleep($requestTime * 1000000); - - $localDisk->put( - "{$directoryName}/{$potentialFilename}", - $remoteDisk->get("remoteIncludeCache/{$hashedUrl}") - ); - $linesOfCode[$i] = '#include "' . $potentialFilename .'"'; - continue; - } - } - - $log->info("remote include cache miss"); - - try - { - $request = new PendingRequest(); - $request->timeout(3); - $response = $request->head($potentialUrl); - } - catch(Exception $e) - { - $errors[] = "/pgetinker.cpp:" . $i + 1 . ":1: error: failed to retrieve {$potentialUrl}"; - $log->info("failed to include remote file: {$potentialUrl} at line: " . $i + 1, [ "message" => $e->getMessage()]); - continue; - } - - if( - !($response->status() >= 200 && $response->status() < 400) || - !str_contains($response->header("Content-Type"), "text/plain") - ) - { - $errors[] = "/pgetinker.cpp:" . $i + 1 . ":1: error: failed to retrieve {$potentialUrl}"; - $log->info("failed to include remote file: {$potentialUrl} at line: " . $i + 1); - continue; - } - - if(intval($response->header("Content-Length")) > 1048576) - { - $errors[] = "/pgetinker.cpp:" . $i + 1 . ":1: error: exceeds 1MB maximum file size"; - $log->info("remote file: {$potentialUrl} exceeds 1MB file size limitation"); - continue; - } - - $log->info("retrieving the body content"); - - try - { - $requestStartTime = microtime(true); - - $request = new PendingRequest(); - $request->timeout(5); - - $response = $request->get($potentialUrl); - - $requestDuration = microtime(true) - $requestStartTime; - } - catch(Exception $e) - { - $errors[] = "/pgetinker.cpp:" . $i + 1 . ":1: error: failed to retrieve {$potentialUrl}"; - $log->info("failed to include remote file: {$potentialUrl} at line: " . $i + 1); - continue; - } - - // check included source for bad things - preg_match_all( - '/\s*#\s*i(nclude|mport)(_next)?\s+["<]((\.{1,2}|\/)[^">]*)[">]/m', - $response->body(), - $match, - PREG_SET_ORDER, - 0 - ); - - if(count($match) > 0) - { - $errors[] = "/pgetinker.cpp:" . $i + 1 . ":1: error: found absolute or relative paths in remote file: {$potentialUrl}"; - $log->info("found absolute or relative paths in remote file: {$potentialUrl}"); - continue; - } - - $log->info("writing remote file to: {$directoryName}/{$potentialFilename}"); - $localDisk->put("{$directoryName}/{$potentialFilename}", $response->body()); - - if(env("REMOTE_INCLUDE_CACHING", false)) - { - $log->info("caching remotely included source file: $potentialFilename"); - $remoteDisk->put("remoteIncludeCache/{$hashedUrl}", $response->body()); - $remoteDisk->put("remoteIncludeCache/{$hashedUrl}.time", $requestDuration); - } - - $linesOfCode[$i] = '#include "' . $potentialFilename .'"'; - continue; - } + Storage::disk("local")->makeDirectory("workspaces"); } - - // bail if we have errors here, no need to invoke the compiler - if(count($errors) > 0) - { - $response = [ - "statusCode" => 400, - "stdout" => "", - "stderr" => implode("\n", $errors), - ]; - $log->info("compilation failed"); - return $response; - } - - $version = "v0.01"; - $workspaceDirectory = $localDisk->path($directoryName); - $thirdPartyDirectory = base_path() . "/third_party"; - - $compilerEnvironment = env("COMPILER_ENVIRONMENT", "local"); - - $log->info("writing linesOfCode to {$directoryName}/pgetinker.cpp"); - $localDisk->put("{$directoryName}/pgetinker.cpp", implode("\n", $linesOfCode)); - - $environmentVariables = []; - $compilerCommand = null; - $linkerCommand = null; - - if($compilerEnvironment === "local") - { - $log->info("preparing compiler environment: {$compilerEnvironment}"); - - $environmentVariables = array_merge($environmentVariables, [ - "EMSDK" => "/opt/emsdk", - "EMSDK_NODE" => "/opt/emsdk/node/16.20.0_64bit/bin/node", - "PATH" => "/bin:/usr/bin:/opt/emsdk:/opt/emsdk/upstream/emscripten", - ]); - - symlink("{$thirdPartyDirectory}/{$version}/include", "{$workspaceDirectory}/include"); - symlink("{$thirdPartyDirectory}/{$version}/lib", "{$workspaceDirectory}/lib"); - symlink("{$thirdPartyDirectory}/emscripten_shell.html", "{$workspaceDirectory}/emscripten_shell.html"); - - $compilerCommand = []; - $linkerCommand = []; - } - - if($compilerEnvironment === "nsjail") - { - $log->info("preparing compiler environment: {$compilerEnvironment}"); - - $nsJailCommand = [ - "nsjail", - "--config", - "{$thirdPartyDirectory}/nsjail-emscripten.cfg", - "-B", - "{$workspaceDirectory}:/user", - "-R", - "{$thirdPartyDirectory}/{$version}/include:/user/include", - "-R", - "{$thirdPartyDirectory}/{$version}/lib:/user/lib", - "-R", - "{$thirdPartyDirectory}/emscripten_shell.html:/user/emscripten_shell.html", - "--", - ]; - - $compilerCommand = $nsJailCommand; - $linkerCommand = $nsJailCommand; - } - - if($compilerCommand === null || $linkerCommand === null) - { - throw new Exception("unknown compiler environment"); - } - - $log->info("preparing compiler command"); - $compilerCommand = array_merge($compilerCommand, [ - "/opt/emsdk/upstream/emscripten/em++", - "-c", - "-I./include/olcPixelGameEngine", - "-I./include/olcPixelGameEngine/extensions", - "-I./include/olcPixelGameEngine/utilities", - "-I./include/olcSoundWaveEngine", - "pgetinker.cpp", - "-o", - "pgetinker.o", - "-std=c++20", - ]); - - $log->info("preparing linker command"); - $linkerCommand = array_merge($linkerCommand, [ - "/opt/emsdk/upstream/emscripten/em++", - "pgetinker.o", - ...$libraries, - "-o", - "pgetinker.html", - "--shell-file", - "./emscripten_shell.html", - "-sASYNCIFY", - "-sALLOW_MEMORY_GROWTH=1", - "-sMAX_WEBGL_VERSION=2", - "-sMIN_WEBGL_VERSION=2", - "-sUSE_LIBPNG=1", - "-sUSE_SDL_MIXER=2", - "-sLLD_REPORT_UNDEFINED", - "-sSINGLE_FILE", - "-std=c++20", - ]); - - $response = null; - - $log->info("invoking the compiler"); - try - { - $compilerProcessResult = Process::env($environmentVariables) - ->path($workspaceDirectory) - ->timeout(10) - ->command($compilerCommand)->run(); - - $log->info("compiler exited with code: " . $compilerProcessResult->exitCode()); - } - catch(Exception $e) - { - $response = [ - "statusCode" => 400, - "stdout" => "", - "stderr" => "compiler timed out. your code is either broken or there's too much of it!", - ]; - } - - if($response == null) - { - $response = [ - "statusCode" => 400, - "stdout" => $this->filterOutput($compilerProcessResult->output()), - "stderr" => $this->filterOutput($compilerProcessResult->errorOutput()), - ]; - - $log->error("compilation failed", [ - "stdout" => $compilerProcessResult->output(), - "stderr" => $compilerProcessResult->errorOutput(), - ]); - } - - // if we're here and $response is still null, the compile stage succeeded, invoke linker - if($response == null) - { - $log->info("invoking the linker"); - - try - { - $linkerProcessResult = Process::env($environmentVariables) - ->path($workspaceDirectory) - ->timeout(10) - ->command($linkerCommand)->run(); - } - catch(Exception $e) - { - $response = [ - "statusCode" => 400, - "stdout" => "", - "stderr" => "linker timed out. your code is either broken or there's too much of it or there's some other bug, report it?", - ]; - } - - if($linkerProcessResult->exitCode() !== 0) - { - $response = [ - "statusCode" => 400, - "stdout" => $this->filterOutput($linkerProcessResult->output()), - "stderr" => $this->filterOutput($linkerProcessResult->errorOutput()), - ]; - - $log->error("linking failed", [ - "stdout" => $linkerProcessResult->output(), - "stderr" => $linkerProcessResult->errorOutput(), - ]); - } - } + $directoryName = "workspaces/" . Str::uuid(); + Storage::disk("local")->makeDirectory($directoryName); - if($response == null) - { - if(!$localDisk->exists("{$directoryName}/pgetinker.html")) - { - $response = [ - "statusCode" => 42069, - "stderr" => "something really bad happened in order for this to occur. contact the administrator", - ]; + Log::debug("Compile: working directory created {$directoryName}"); - Log::debug("Compile: failed beyond the linker stage", $response); - } - } + $compiler = new Compiler(); + $compiler + ->setCode($code) + ->setWorkingDirectory(Storage::disk("local")->path($directoryName)); - if($response == null) + if($compiler->build()) { - // if we've made it here, SUCCESS! - $html = $localDisk->get("{$directoryName}/pgetinker.html"); - - if(env("COMPILER_CACHING", false)) - { - $remoteDisk->put("compilerCache/{$hashedCode}", $html); - } - - $localDisk->deleteDirectory($directoryName); + Storage::put( + "compilerCache/{$hashedCode}", + $compiler->getHtml() + ); - $response = [ + return [ "statusCode" => 200, "hash" => $hashedCode, - "html" => $html, + "html" => $compiler->getHtml(), + "stdout" => $compiler->getOutput(), + "stderr" => $compiler->getErrorOutput(), ]; - - Log::info("Compile: finished successfully"); - } - - if($response["statusCode"] != 200) - { - // it's possible that the local disk and remote disk - // are the same - if(get_class($localDisk) != get_class($remoteDisk)) - { - Log::info("uploading files to remote disk."); - // get the local files - $files = $localDisk->files($directoryName); - - // create the remote directory - $remoteDisk->makeDirectory($directoryName); - - for($i = 0; $i < count($files); $i++) - { - // copy the file from the localDisk to the remoteDisk - $remoteDisk->put( - $files[$i], - $localDisk->get($files[$i]) - ); - } - - // remove the local files - $localDisk->deleteDirectory($directoryName); - } - Log::info("Compile: finished disgracefully"); } - - return $response; + + return [ + "statusCode" => 400, + "hash" => $hashedCode, + "stdout" => $compiler->getOutput(), + "stderr" => $compiler->getErrorOutput(), + ]; } function filterOutput($text) diff --git a/composer.json b/composer.json index 1c0ddd0..8ce4c11 100644 --- a/composer.json +++ b/composer.json @@ -24,10 +24,14 @@ "spatie/laravel-ignition": "^2.4" }, "autoload": { + "files": [ + "pgetinker/Utils.php" + ], "psr-4": { "App\\": "app/", "Database\\Factories\\": "database/factories/", - "Database\\Seeders\\": "database/seeders/" + "Database\\Seeders\\": "database/seeders/", + "PGEtinker\\": "pgetinker/" } }, "autoload-dev": { diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 8c27d34..7bc349b 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -6,5 +6,9 @@ su -c "php artisan cache:clear" -s /bin/bash www-data su -c "php artisan view:clear" -s /bin/bash www-data su -c "php artisan route:clear" -s /bin/bash www-data +# nsjail/cgroup stuff +cgcreate -a www-data:www-data -g memory,pids,cpu:pgetinker-compile +sudo chown www-data:root /sys/fs/cgroup/cgroup.procs + # the server apache2-foreground diff --git a/pgetinker/Compiler.php b/pgetinker/Compiler.php new file mode 100644 index 0000000..130f107 --- /dev/null +++ b/pgetinker/Compiler.php @@ -0,0 +1,550 @@ +logger = new Logger("compiler"); + return $this; + } + + public function setCode(string $code) + { + $this->code = explode("\n", $code); + return $this; + } + + public function setWorkingDirectory(string $workingDirectory) + { + $this->workingDirectory = $workingDirectory; + return $this; + } + + public function getOutput() + { + return implode("\n", $this->output); + } + + public function getErrorOutput() + { + return implode("\n", $this->errors); + } + + public function getHtml() + { + return $this->html; + } + + private function processCodeAbsoluteOrRelativePaths($index) + { + // filter include macros with an absolute or relative path, naughty naughty + preg_match( + '/^\s*#\s*i(nclude|mport)(_next)?\s+["<]((\.{1,2}|\/)[^">]*)[">]/', + $this->code[$index], + $match, + PREG_OFFSET_CAPTURE, + 0 + ); + + if(count($match) > 0) + { + $this->errors[] = "/pgetinker.cpp:" . $index + 1 . ":1: error: absolute and relative includes are not allowed."; + $this->logger->info("found absolute or relative path at line " . $index + 1); + return true; + } + + return false; + } + + private function processCodeDetectImplementationMacros($index) + { + $libraryMap = [ + 'OLC_PGE_APPLICATION' => 'olcPixelGameEngine.o', + 'OLC_SOUNDWAVE_ENGINE' => 'olcSoundWaveEngine.o', + 'OLC_PGEX_GRAPHICS2D' => 'olcPGEX_Graphics2D.o', + 'OLC_PGEX_GRAPHICS3D' => 'olcPGEX_Graphics3D.o', + 'OLC_PGEX_POPUPMENU' => 'olcPGEX_PopUpMenu.o', + 'OLC_PGEX_QUICKGUI' => 'olcPGEX_QuickGUI.o', + 'OLC_PGEX_RAYCASTWORLD' => 'olcPGEX_RayCastWorld.o', + 'OLC_PGEX_SOUND' => 'olcPGEX_Sound.o', + 'OLC_PGEX_SPLASHSCREEN' => 'olcPGEX_SplashScreen.o', + 'OLC_PGEX_TRANSFORMEDVIEW' => 'olcPGEX_TransformedView.o', + 'OLC_PGEX_WIREFRAME' => 'olcPGEX_Wireframe.o', + ]; + + // filter macros to detect implementation #define + if(str_contains($this->code[$index], "#define")) + { + $foundImplementationMacro = false; + foreach($libraryMap as $macro => $objectFileName) + { + if(str_contains($this->code[$index], $macro)) + { + // blank the line + $this->code[$index] = ""; + + // indicate that we use this library + $this->linkerInputFiles[] = "./lib/{$objectFileName}"; + + $this->logger->info("Found implementation macro: {$macro}"); + $foundImplementationMacro = true; + break; + } + } + + if($foundImplementationMacro) + return true; + } + + return false; + } + + private function processCodeRemoteInclude($index) + { + preg_match( + '/^\s*#\s*i(nclude|mport)(_next)?\s+["<](https?:\/\/(.*)[^">]*)[">]/', + $this->code[$index], + $match, + PREG_OFFSET_CAPTURE, + 0 + ); + + if(count($match) > 0) + { + $this->logger->info("found a potential url for remote include"); + + $potentialUrl = $match[3][0]; + $potentialFilename = basename($match[3][0]); + $hashedUrl = hash("sha256", $potentialUrl); + + if(env("COMPILER_REMOTE_INCLUDE_CACHING", false)) + { + // if we have a cached version of the url's contents, don't pull it + if( + Storage::fileExists("remoteIncludeCache/{$hashedUrl}") && + Storage::fileExists("remoteIncludeCache/{$hashedUrl}.time") + ) + { + $this->logger->info("remote include cache hit"); + + $requestTime = floatval(Storage::get("remoteIncludeCache/{$hashedUrl}.time")); + + // just because it's cached, doesn't mean you get to compile faster! + usleep($requestTime * 1000000); + + file_put_contents( + "{$this->workingDirectory}/{$potentialFilename}", + Storage::get("remoteIncludeCache/{$hashedUrl}") + ); + + $this->code[$index] = '#include "' . $potentialFilename .'"'; + return true; + } + } + + $this->logger->info("remote include cache miss"); + + try + { + $request = new PendingRequest(); + $request->timeout(3); + $response = $request->head($potentialUrl); + } + catch(Exception $e) + { + $this->errors[] = "/pgetinker.cpp:" . $index + 1 . ":1: error: failed to retrieve {$potentialUrl}"; + $this->logger->info("failed to include remote file: {$potentialUrl} at line: " . $index + 1, [ "message" => $e->getMessage()]); + return true; + } + + if( + !($response->status() >= 200 && $response->status() < 400) || + !str_contains($response->header("Content-Type"), "text/plain") + ) + { + $this->errors[] = "/pgetinker.cpp:" . $index + 1 . ":1: error: failed to retrieve {$potentialUrl}"; + $this->logger->info("failed to include remote file: {$potentialUrl} at line: " . $index + 1); + return true; + } + + if(intval($response->header("Content-Length")) > 1048576) + { + $this->errors[] = "/pgetinker.cpp:" . $index + 1 . ":1: error: exceeds 1MB maximum file size"; + $this->logger->info("remote file: {$potentialUrl} exceeds 1MB file size limitation"); + return true; + } + + $this->logger->info("retrieving the body content"); + + try + { + $requestStartTime = microtime(true); + + $request = new PendingRequest(); + $request->timeout(5); + + $response = $request->get($potentialUrl); + + $requestDuration = microtime(true) - $requestStartTime; + } + catch(Exception $e) + { + $this->errors[] = "/pgetinker.cpp:" . $index + 1 . ":1: error: failed to retrieve {$potentialUrl}"; + $this->logger->info("failed to include remote file: {$potentialUrl} at line: " . $index + 1); + return true; + } + + // check included source for bad things + preg_match_all( + '/\s*#\s*i(nclude|mport)(_next)?\s+["<]((\.{1,2}|\/)[^">]*)[">]/m', + $response->body(), + $match, + PREG_SET_ORDER, + 0 + ); + + if(count($match) > 0) + { + $this->errors[] = "/pgetinker.cpp:" . $index + 1 . ":1: error: found absolute or relative paths in remote file: {$potentialUrl}"; + $this->logger->info("found absolute or relative paths in remote file: {$potentialUrl}"); + return true; + } + + $this->logger->info("writing remote file to: {$this->workingDirectory}/{$potentialFilename}"); + file_put_contents( + "{$this->workingDirectory}/{$potentialFilename}", + $response->body() + ); + + if(env("COMPILER_REMOTE_INCLUDE_CACHING", false)) + { + $this->logger->info("caching remotely included source file: $potentialFilename"); + Storage::put("remoteIncludeCache/{$hashedUrl}", $response->body()); + Storage::put("remoteIncludeCache/{$hashedUrl}.time", $requestDuration); + } + + $this->code[$index] = '#include "' . $potentialFilename .'"'; + return true; + } + } + + private function processCode() + { + $this->logger->info("begin processing code"); + $startTime = microtime(true); + + for($i = 0; $i < count($this->code); $i++) + { + $endTime = microtime(true); + $duration = $endTime - $startTime; + + if($duration > intval(env("COMPILER_CODE_PROCESSING_TIMEOUT", 5))) + { + $this->errors[] = "/pgetinker.cpp:" . $i . ":1: error: took too long to process your code, stopped here"; + return false; + } + + if($this->processCodeAbsoluteOrRelativePaths($i)) + continue; + + if($this->processCodeDetectImplementationMacros($i)) + continue; + + if($this->processCodeRemoteInclude($i)) + continue; + } + + $this->logger->info("finished processing code"); + + return (count($this->errors) == 0); + } + + private function prepareEnvironment() + { + $version = "v0.01"; + + $compilerEnvironment = env("COMPILER_ENVIRONMENT", "local"); + + $this->logger->info("writing linesOfCode to {$this->workingDirectory}/pgetinker.cpp"); + file_put_contents( + "{$this->workingDirectory}/pgetinker.cpp", + implode("\n", $this->code) + ); + + if($compilerEnvironment === "local") + { + $this->logger->info("preparing compiler environment: {$compilerEnvironment}"); + + $this->environmentVariables = array_merge($this->environmentVariables, [ + "EMSDK" => "/opt/emsdk", + "EMSDK_NODE" => "/opt/emsdk/node/16.20.0_64bit/bin/node", + "PATH" => "/bin:/usr/bin:/opt/emsdk:/opt/emsdk/upstream/emscripten", + ]); + + symlink(base_path() . "/third_party/{$version}/include", "{$this->workingDirectory}/include"); + symlink(base_path() . "/third_party/{$version}/lib", "{$this->workingDirectory}/lib"); + symlink(base_path() . "/third_party/emscripten_shell.html", "{$this->workingDirectory}/emscripten_shell.html"); + } + + if($compilerEnvironment === "nsjail") + { + $this->logger->info("preparing compiler environment: {$compilerEnvironment}"); + + $nsJailCommand = [ + "nsjail", + "--config", + base_path() . "/third_party/nsjail-emscripten.cfg", + "-B", + "{$this->workingDirectory}:/user", + "-R", + base_path() . "/third_party/{$version}/include:/user/include", + "-R", + base_path() . "/third_party/{$version}/lib:/user/lib", + "-R", + base_path() . "/third_party/emscripten_shell.html:/user/emscripten_shell.html", + "--", + ]; + + $this->compilerCommand = $nsJailCommand; + $this->linkerCommand = $nsJailCommand; + } + + $this->logger->info("preparing compiler command"); + $this->compilerCommand = array_merge($this->compilerCommand, [ + "/opt/emsdk/upstream/emscripten/em++", + "-c", + "-I./include/olcPixelGameEngine", + "-I./include/olcPixelGameEngine/extensions", + "-I./include/olcPixelGameEngine/utilities", + "-I./include/olcSoundWaveEngine", + "pgetinker.cpp", + "-o", + "pgetinker.o", + "-std=c++20", + ]); + + $this->logger->info("preparing linker command"); + $this->linkerCommand = array_merge($this->linkerCommand, [ + "/opt/emsdk/upstream/emscripten/em++", + "pgetinker.o", + ...$this->linkerInputFiles, + "-o", + "pgetinker.html", + "--shell-file", + "./emscripten_shell.html", + "-sASYNCIFY", + "-sALLOW_MEMORY_GROWTH=1", + "-sMAX_WEBGL_VERSION=2", + "-sMIN_WEBGL_VERSION=2", + "-sUSE_LIBPNG=1", + "-sUSE_SDL_MIXER=2", + "-sLLD_REPORT_UNDEFINED", + "-sSINGLE_FILE", + "-std=c++20", + ]); + } + + private function compile() + { + $this->logger->info("invoking the compiler"); + + $didTheThingSuccessfully = false; + + try + { + $compilerProcessResult = Process::env($this->environmentVariables) + ->path($this->workingDirectory) + ->timeout(intval(env("COMPILER_TIMEOUT", 10))) + ->command($this->compilerCommand)->run(); + + $this->logger->info("compiler exited with code: " . $compilerProcessResult->exitCode()); + + $this->compilerExitCode = $compilerProcessResult->exitCode(); + + $didTheThingSuccessfully = ($this->compilerExitCode == 0); + + $this->output = array_merge( + $this->output, + explode("\n", $compilerProcessResult->output()) + ); + + $this->errors = array_merge( + $this->errors, + explode("\n", $compilerProcessResult->errorOutput()) + ); + + if($this->compilerExitCode == 137) + { + $this->errors[] = "Compiler Killed (SIGTERM)"; + } + } + catch(Exception $e) + { + $this->errors[] = "compiler timed out. your code is either broken or there's too much of it!"; + $didTheThingSuccessfully = false; + } + + return $didTheThingSuccessfully; + } + + private function link() + { + $this->logger->info("invoking the linker"); + + $didTheThingSuccessfully = false; + + try + { + $linkerProcessResult = Process::env($this->environmentVariables) + ->path($this->workingDirectory) + ->timeout(intval(env("COMPILER_TIMEOUT", 10))) + ->command($this->linkerCommand)->run(); + + $this->logger->info("compiler exited with code: " . $linkerProcessResult->exitCode()); + + $this->linkerExitCode = $linkerProcessResult->exitCode(); + + $didTheThingSuccessfully = ($this->linkerExitCode == 0); + + $this->output = array_merge( + $this->output, + explode("\n", $linkerProcessResult->output()) + ); + + $this->errors = array_merge( + $this->errors, + explode("\n", $linkerProcessResult->errorOutput()) + ); + + if($this->compilerExitCode == 137) + { + $this->errors[] = "Linker Killed (SIGTERM)"; + } + } + catch(Exception $e) + { + $this->errors[] = "linker timed out. your code is either broken or there's too much of it!"; + $didTheThingSuccessfully = false; + } + + return $didTheThingSuccessfully; + } + + private function cleanUp() + { + $this->logger->info("cleanUp called"); + + $this->logger->info("OUTPUT:\n\n" . $this->getOutput() . "\n\nERROR:\n\n" . $this->getErrorOutput()); + + if(env("FILESYSTEM_DISK") == "s3") + { + // convert workingDirectory to laravel disk path + $prefix = dirname($this->workingDirectory); + $this->workingDirectory = str_replace("{$prefix}/", "", $this->workingDirectory); + + Log::info("uploading files to remote disk."); + + // get the local files + $files = Storage::disk("local")->files($this->workingDirectory); + + // create the s3 directory + Storage::makeDirectory($this->workingDirectory); + + for($i = 0; $i < count($files); $i++) + { + // copy the file from the localDisk to the s3 + Storage::put( + $files[$i], + Storage::disk("local")->get($files[$i]) + ); + } + + // remove the local files + Storage::disk("local")->deleteDirectory($this->workingDirectory); + } + } + + public function build() + { + if(!file_exists($this->workingDirectory) || !is_dir($this->workingDirectory)) + throw new Exception("Working Directory Inaccessible. Did you set one?"); + + $logHandler = new StreamHandler("{$this->workingDirectory}/compiler.log"); + $logHandler->setFormatter(new LineFormatter(null, null, true, true)); + + $this->logger->setHandlers([$logHandler]); + + if(!$this->processCode()) + { + $this->cleanUp(); + return false; + } + + $this->prepareEnvironment(); + + if(!$this->compile()) + { + $this->cleanUp(); + return false; + } + + if(!$this->link()) + { + $this->cleanUp(); + return false; + } + + if(file_exists("{$this->workingDirectory}/pgetinker.html")) + { + $this->html = file_get_contents("{$this->workingDirectory}/pgetinker.html"); + + // convert workingDirectory to laravel disk path + $prefix = dirname($this->workingDirectory); + $this->workingDirectory = str_replace("{$prefix}/", "", $this->workingDirectory); + + Storage::disk("local")->deleteDirectory($this->workingDirectory); + + Log::info("Compile: finished successfully"); + + return true; + } + + $this->cleanUp(); + return false; + } +} diff --git a/pgetinker/Utils.php b/pgetinker/Utils.php new file mode 100644 index 0000000..8f8fd8f --- /dev/null +++ b/pgetinker/Utils.php @@ -0,0 +1,75 @@ + tokens + */ + foreach([ + "(","[","{",")","]","}",",","-","+","*","=" + ] as $token) + { + $code = str_replace($token, " {$token} ", $code); + } + + /** + * thanks Bixxy for the general idea here. + */ + $tokens = token_get_all(" 1) + $text = " "; + + } + + // if, for any reason we reach here, add it to the code we wanna hash + $cppcode .= $text; + continue; + } + else + { + // any other token is passed through, as is. + $cppcode .= $token; + } + } + + // take off multiple new lines left over + $cppcode = preg_replace('/\n\s*\n/', "\n", $cppcode); + + return hash("sha256", $cppcode); +} + diff --git a/tests/Feature/CodeControllerHashTest.php b/tests/Feature/CodeControllerHashTest.php deleted file mode 100644 index 8c74546..0000000 --- a/tests/Feature/CodeControllerHashTest.php +++ /dev/null @@ -1,56 +0,0 @@ -assertTrue($c->hashCode("(a)") == $c->hashCode("( a)")); - $this->assertTrue($c->hashCode("(a)") == $c->hashCode("(a )")); - } - - public function test_comma_hash(): void - { - $c = new CodeController(); - $this->assertTrue($c->hashCode(",") == $c->hashCode(" ,")); - $this->assertTrue($c->hashCode(",") == $c->hashCode(", ")); - } - - public function test_comment_hash(): void - { - $c = new CodeController(); - $this->assertTrue($c->hashCode("// this is a comment") == $c->hashCode("// this is a longer comment")); - } - - public function test_string_literal_hash(): void - { - $c = new CodeController(); - $this->assertFalse($c->hashCode('"this is a string"') == $c->hashCode('"this is a longer string"')); - } - - public function test_one_line_macro(): void - { - $c = new CodeController(); - $this->assertFalse($c->hashCode("#define SOME_MACRO") == $c->hashCode("#define SOME_NEW_MACRO")); - } - - public function test_math_operators(): void - { - $c = new CodeController(); - foreach(["-", "+", "*"] as $operator) - { - $this->assertTrue($c->hashCode("{$operator}") == $c->hashCode(" {$operator}")); - $this->assertTrue($c->hashCode("{$operator}") == $c->hashCode("{$operator} ")); - } - } - -} diff --git a/tests/Feature/CompilerTest.php b/tests/Feature/CompilerTest.php index 6fe920e..5d670b5 100644 --- a/tests/Feature/CompilerTest.php +++ b/tests/Feature/CompilerTest.php @@ -2,19 +2,92 @@ namespace Tests\Feature; -// use Illuminate\Foundation\Testing\RefreshDatabase; - -use App\Http\Controllers\CodeController; +use Illuminate\Support\Facades\Storage; +use PGEtinker\Compiler; use Tests\TestCase; class CompilerTest extends TestCase { - public function test_compiler_compiles_hello_world(): void + + + public function test_compiler_builds_hello_world(): void + { + if(Storage::disk("local")->exists(__FUNCTION__)) + Storage::disk("local")->deleteDirectory(__FUNCTION__); + + Storage::disk("local")->makeDirectory(__FUNCTION__); + $workingDirectory = Storage::disk("local")->path(__FUNCTION__); + $testSourceDirectory = __DIR__ . "/compiler-test-source"; + + $compiler = new Compiler(); + $compiler->setWorkingDirectory($workingDirectory); + $compiler->setCode(file_get_contents("{$testSourceDirectory}/hello-world.cpp")); + + $this->assertTrue($compiler->build()); + } + + public function test_compiler_build_timeout(): void + { + if(Storage::disk("local")->exists(__FUNCTION__)) + Storage::disk("local")->deleteDirectory(__FUNCTION__); + + Storage::disk("local")->makeDirectory(__FUNCTION__); + $workingDirectory = Storage::disk("local")->path(__FUNCTION__); + $testSourceDirectory = __DIR__ . "/compiler-test-source"; + + $compiler = new Compiler(); + $compiler->setWorkingDirectory($workingDirectory); + $compiler->setCode(file_get_contents("{$testSourceDirectory}/ice-timeout.cpp")); + + $this->assertFalse($compiler->build()); + } + + public function test_compiler_builds_example(): void { - $response = $this->post("/api/compile", [ - "code" => '#include int main() { printf("Hello, World\n"); return 0; }' - ]); + if(Storage::disk("local")->exists(__FUNCTION__)) + Storage::disk("local")->deleteDirectory(__FUNCTION__); + + Storage::disk("local")->makeDirectory(__FUNCTION__); + $workingDirectory = Storage::disk("local")->path(__FUNCTION__); + $testSourceDirectory = __DIR__ . "/compiler-test-source"; + + $compiler = new Compiler(); + $compiler->setWorkingDirectory($workingDirectory); + $compiler->setCode(file_get_contents("{$testSourceDirectory}/example.cpp")); - $response->assertStatus(200); + $this->assertTrue($compiler->build()); } + + public function test_compiler_absolute_and_relative_include_trap(): void + { + if(Storage::disk("local")->exists(__FUNCTION__)) + Storage::disk("local")->deleteDirectory(__FUNCTION__); + + Storage::disk("local")->makeDirectory(__FUNCTION__); + $workingDirectory = Storage::disk("local")->path(__FUNCTION__); + $testSourceDirectory = __DIR__ . "/compiler-test-source"; + + $compiler = new Compiler(); + $compiler->setWorkingDirectory($workingDirectory); + $compiler->setCode(file_get_contents("{$testSourceDirectory}/absolute-or-relative.cpp")); + + $this->assertFalse($compiler->build()); + } + + public function test_remote_includes(): void + { + if(Storage::disk("local")->exists(__FUNCTION__)) + Storage::disk("local")->deleteDirectory(__FUNCTION__); + + Storage::disk("local")->makeDirectory(__FUNCTION__); + $workingDirectory = Storage::disk("local")->path(__FUNCTION__); + $testSourceDirectory = __DIR__ . "/compiler-test-source"; + + $compiler = new Compiler(); + $compiler->setWorkingDirectory($workingDirectory); + $compiler->setCode(file_get_contents("{$testSourceDirectory}/remote-includes.cpp")); + + $this->assertTrue($compiler->build()); + } + } diff --git a/tests/Feature/UtilsHashCodeTest.php b/tests/Feature/UtilsHashCodeTest.php new file mode 100644 index 0000000..00de674 --- /dev/null +++ b/tests/Feature/UtilsHashCodeTest.php @@ -0,0 +1,49 @@ +assertTrue(hashCode("(a)") == hashCode("( a)")); + $this->assertTrue(hashCode("(a)") == hashCode("(a )")); + } + + public function test_comma_hash(): void + { + $this->assertTrue(hashCode(",") == hashCode(" ,")); + $this->assertTrue(hashCode(",") == hashCode(", ")); + } + + public function test_comment_hash(): void + { + $this->assertTrue(hashCode("// this is a comment") == hashCode("// this is a longer comment")); + } + + public function test_string_literal_hash(): void + { + $this->assertFalse(hashCode('"this is a string"') == hashCode('"this is a longer string"')); + } + + public function test_one_line_macro(): void + { + $this->assertFalse(hashCode("#define SOME_MACRO") == hashCode("#define SOME_NEW_MACRO")); + } + + public function test_math_operators(): void + { + foreach(["-", "+", "*"] as $operator) + { + $this->assertTrue(hashCode("{$operator}") == hashCode(" {$operator}")); + $this->assertTrue(hashCode("{$operator}") == hashCode("{$operator} ")); + } + } + +} diff --git a/tests/Feature/compiler-test-source/absolute-or-relative.cpp b/tests/Feature/compiler-test-source/absolute-or-relative.cpp new file mode 100644 index 0000000..6b85024 --- /dev/null +++ b/tests/Feature/compiler-test-source/absolute-or-relative.cpp @@ -0,0 +1,11 @@ +#include + +#include "/etc/hosts" + +#include "../some/totally/ok/include.h" + +int main() +{ + printf("Hello,World\n"); + return 0; +} diff --git a/tests/Feature/compiler-test-source/example.cpp b/tests/Feature/compiler-test-source/example.cpp new file mode 100644 index 0000000..307d800 --- /dev/null +++ b/tests/Feature/compiler-test-source/example.cpp @@ -0,0 +1,76 @@ +#define OLC_PGE_APPLICATION +#include "olcPixelGameEngine.h" + +#if defined(__EMSCRIPTEN__) +#include +#define FILE_RESOLVE(url, file) emscripten_wget(url, file); emscripten_sleep(0) +#else +#define FILE_RESOLVE(url, file) +#endif + +// Override base class with your custom functionality +class Example : public olc::PixelGameEngine +{ +public: + Example() + { + // Name your application + sAppName = "Example"; + } + +public: + bool OnUserCreate() override + { + // Called once at the start, so create things here + + // built with emscripten, maps the url to the virtual filesystem and makes it + // available to the standard C/C++ file i/o functions without change! + // + // built with any other native toolchain, the macro does nothing and all file + // access is done just as it would in any other normal scenario. + FILE_RESOLVE("https://i.imgur.com/KdWjkwC.png", "assets/gfx/broken.png"); + + renBroken.Load("assets/gfx/broken.png"); + + color = RandomColor(); + return true; + } + + bool OnUserUpdate(float fElapsedTime) override + { + // Called once per frame, draws random coloured pixels + if(GetMouse(0).bPressed) + color = RandomColor(); + + Clear(color); + DrawRect(0,0,ScreenWidth()-1, ScreenHeight()-1, olc::YELLOW); + DrawString(6, 6, "Hello, PGE", olc::BLACK); + DrawString(5, 5, "Hello, PGE", olc::WHITE); + DrawString(6, 26, "Mouse position SHOULD match\nclosely to the circle.\n\nYellow borders should ALWAYS\nbe visible\n\nLEFT MOUSE to change color.", olc::BLACK); + DrawString(5, 25, "Mouse position SHOULD match\nclosely to the circle.\n\nYellow borders should ALWAYS\nbe visible\n\nLEFT MOUSE to change color.", olc::WHITE); + + DrawSprite(5, 100, renBroken.Sprite()); + + DrawString(6, 221, GetMousePos().str(), olc::BLACK); + DrawString(5, 220, GetMousePos().str(), olc::WHITE); + FillCircle(GetMousePos(), 3, olc::RED); + Draw(GetMousePos(), olc::WHITE); + return true; + } + + olc::Pixel RandomColor() + { + return olc::Pixel(rand() % 128, rand() % 128, rand() % 128); + } + + olc::Pixel color; + olc::Renderable renBroken; +}; + +int main() +{ + Example demo; + if (demo.Construct(256, 240, 2, 2)) + demo.Start(); + return 0; +} \ No newline at end of file diff --git a/tests/Feature/compiler-test-source/hello-world.cpp b/tests/Feature/compiler-test-source/hello-world.cpp new file mode 100644 index 0000000..dcbcca1 --- /dev/null +++ b/tests/Feature/compiler-test-source/hello-world.cpp @@ -0,0 +1,7 @@ +#include + +int main() +{ + printf("Hello,World\n"); + return 0; +} diff --git a/tests/Feature/compiler-test-source/ice-timeout.cpp b/tests/Feature/compiler-test-source/ice-timeout.cpp new file mode 100644 index 0000000..470a41c --- /dev/null +++ b/tests/Feature/compiler-test-source/ice-timeout.cpp @@ -0,0 +1,6 @@ +#include +#include +#include +#include +#include +std::make_index_sequence<~0U>{}; \ No newline at end of file diff --git a/tests/Feature/compiler-test-source/remote-includes.cpp b/tests/Feature/compiler-test-source/remote-includes.cpp new file mode 100644 index 0000000..661877e --- /dev/null +++ b/tests/Feature/compiler-test-source/remote-includes.cpp @@ -0,0 +1 @@ +#include "https://raw.githubusercontent.com/Moros1138/PGEtinker/main/resources/example.cpp" diff --git a/third_party/nsjail-emscripten.cfg b/third_party/nsjail-emscripten.cfg index 3c1c6b4..11f195d 100644 --- a/third_party/nsjail-emscripten.cfg +++ b/third_party/nsjail-emscripten.cfg @@ -26,6 +26,26 @@ gidmap { inside_id: "10240" } +detect_cgroupv2: true + +# for cgroups v1: +# must run following as root during system startup +# cgcreate -a $USER:$USER -g memory,pids,cpu:pgetinker-compile +cgroup_mem_parent: "pgetinker-compile" +cgroup_pids_parent: "pgetinker-compile" +cgroup_net_cls_parent: "pgetinker-compile" +cgroup_cpu_parent: "pgetinker-compile" + +cgroup_mem_max: 1342177280 # 1.25 GiB +cgroup_pids_max: 72 +cgroup_cpu_ms_per_sec: 1000 + +# for cgroups v2: +# must run following as root during system startup +# cgcreate -a $USER:$USER -g memory,pids,cpu:pgetinker-compile +# sudo chown $USER:root /sys/fs/cgroup/cgroup.procs +cgroupv2_mount: "/sys/fs/cgroup/pgetinker-compile" + mount { src: "/bin" dst: "/bin" From 0adc8846432c9e40d2d5dbce0d9434623ffd253f Mon Sep 17 00:00:00 2001 From: Moros Smith Date: Sat, 11 May 2024 21:38:33 -0400 Subject: [PATCH 3/3] rip off the docker band-aid --- .dockerignore | 10 --- .../workflows/build-and-push-dockerhub.yml | 42 ---------- Dockerfile | 78 ------------------- docker/000-default.conf | 14 ---- docker/docker-entrypoint.sh | 14 ---- docker/php/conf.d/opcache.ini | 9 --- 6 files changed, 167 deletions(-) delete mode 100644 .dockerignore delete mode 100644 .github/workflows/build-and-push-dockerhub.yml delete mode 100644 Dockerfile delete mode 100644 docker/000-default.conf delete mode 100644 docker/docker-entrypoint.sh delete mode 100644 docker/php/conf.d/opcache.ini diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 9d9f38f..0000000 --- a/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -.git -.env -database/database.sqlite -node_modules -vendor -docker-compose.yml -storage/app/workspaces -storage/app/compilerCache -storage/logs/laravel.log -tests/Feature/test_* diff --git a/.github/workflows/build-and-push-dockerhub.yml b/.github/workflows/build-and-push-dockerhub.yml deleted file mode 100644 index ae1ef86..0000000 --- a/.github/workflows/build-and-push-dockerhub.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Dockerhub -on: workflow_dispatch - -jobs: - dockerhub-build: - runs-on: self-hosted - - steps: - - name: 'Cleanup build folder' - run: | - ls -la ./ - rm -rf ./* || true - rm -rf ./.??* || true - ls -la ./ - - - name: Checkout - uses: actions/checkout@v4 - - - name: Get Commit Hash - id: vars - run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Add "VERSION" hashes to .env - run: echo -e "VERSION=${{ steps.vars.outputs.sha }}\nVITE_VERSION=${{ steps.vars.outputs.sha }}" >> .env.example - - - name: Copy environment file and set version - run: cp .env.example .env - - - name: Login to DockerHub - run: docker login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build Docker Image - run: docker buildx build -t ${{ secrets.DOCKERHUB_USERNAME }}/pgetinker:${{ steps.vars.outputs.sha }} -t ${{ secrets.DOCKERHUB_USERNAME }}/pgetinker:latest . - - - name: Push Docker Image - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/pgetinker:${{ steps.vars.outputs.sha }} && docker push ${{ secrets.DOCKERHUB_USERNAME }}/pgetinker:latest - - - name: Clean up Docker - run: docker rmi ${{ secrets.DOCKERHUB_USERNAME }}/pgetinker:latest && docker rmi ${{ secrets.DOCKERHUB_USERNAME }}/pgetinker:${{ steps.vars.outputs.sha }} - - - name: Logout of DockerHub - run: docker logout diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c8f9f5e..0000000 --- a/Dockerfile +++ /dev/null @@ -1,78 +0,0 @@ -FROM composer:2.7.1 as buildComposer -COPY . /app/ -RUN mv .env.example .env -RUN composer install --prefer-dist --no-dev --optimize-autoloader --no-interaction - -FROM node:21-bookworm-slim as buildNode -COPY --from=buildComposer /app /usr/src/app - -WORKDIR /usr/src/app -RUN npm install && npm run build - -FROM php:8.3-apache-bookworm as production - -ENV APP_ENV=production -ENV APP_DEBUG=false -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get -y update && \ - apt-get install -y \ - micro \ - libpq-dev \ - python3 \ - autoconf \ - bison \ - flex \ - gcc \ - g++ \ - git \ - libprotobuf-dev \ - libnl-route-3-dev \ - libtool \ - make \ - pkg-config \ - cgroup-tools \ - protobuf-compiler && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR / - -RUN git clone https://github.com/google/nsjail.git - -RUN cd /nsjail && \ - make && \ - mv /nsjail/nsjail /bin && \ - rm -rf -- /nsjail - -WORKDIR /var/www/html - -RUN docker-php-ext-configure opcache --enable-opcache && \ - docker-php-ext-install pdo pdo_mysql && \ - docker-php-ext-install pdo pdo_pgsql - -COPY docker/php/conf.d/opcache.ini /usr/local/etc/php/conf.d/opcache.ini - -COPY --from=buildNode /usr/src/app /var/www/html -COPY docker/000-default.conf /etc/apache2/sites-available/000-default.conf -COPY docker/docker-entrypoint.sh /docker-entrypoint.sh - -WORKDIR /opt - -RUN git clone https://github.com/emscripten-core/emsdk.git && \ - chown -R www-data:www-data emsdk - -WORKDIR /opt/emsdk - -RUN su -c "bash emsdk install 3.1.56" -s /bin/bash www-data && \ - su -c "bash emsdk activate 3.1.56" -s /bin/bash www-data - -WORKDIR /var/www/html - -RUN chmod 755 -R /var/www/html/storage/ && \ - chown -R www-data:www-data /var/www/ && \ - su -c "bash build-libs.sh" -s /bin/bash www-data && \ - a2enmod rewrite - -ENTRYPOINT [ "/bin/bash" ] - -CMD [ "/docker-entrypoint.sh" ] diff --git a/docker/000-default.conf b/docker/000-default.conf deleted file mode 100644 index fc6eb7c..0000000 --- a/docker/000-default.conf +++ /dev/null @@ -1,14 +0,0 @@ - - - ServerAdmin moros1183@gmail.com - DocumentRoot /var/www/html/public/ - - - AllowOverride All - Require all granted - - - ErrorLog ${APACHE_LOG_DIR}/error.log - CustomLog ${APACHE_LOG_DIR}/access.log combined - - diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh deleted file mode 100644 index 7bc349b..0000000 --- a/docker/docker-entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/bash - -# refresh the cache at startup -su -c "php artisan config:clear" -s /bin/bash www-data -su -c "php artisan cache:clear" -s /bin/bash www-data -su -c "php artisan view:clear" -s /bin/bash www-data -su -c "php artisan route:clear" -s /bin/bash www-data - -# nsjail/cgroup stuff -cgcreate -a www-data:www-data -g memory,pids,cpu:pgetinker-compile -sudo chown www-data:root /sys/fs/cgroup/cgroup.procs - -# the server -apache2-foreground diff --git a/docker/php/conf.d/opcache.ini b/docker/php/conf.d/opcache.ini deleted file mode 100644 index 88cb7ce..0000000 --- a/docker/php/conf.d/opcache.ini +++ /dev/null @@ -1,9 +0,0 @@ -[opcache] -opcache.enable=1 -opcache.revalidate_freq=0 -opcache.validate_timestamps=0 -opcache.max_accelerated_files=10000 -opcache.memory_consumption=192 -opcache.max_wasted_percentage=10 -opcache.interned_strings_buffer=16 -opcache.jit_buffer_size=100M