From 7ecc3fe4135abdf52dc8676538bc8ed3b069baee Mon Sep 17 00:00:00 2001 From: Nebual Date: Sun, 11 Dec 2022 04:14:18 -0800 Subject: [PATCH] New Feature: Mod Pack 7z generation + hosting (Valheim) --- .gitignore | 1 + game-api/package.json | 1 + game-api/src/games/common-helpers.js | 59 ++++++++++++++++++++++++ game-api/src/games/docker.js | 7 ++- game-api/src/games/test.js | 10 ++++ game-api/src/games/valheim.js | 51 +++++++++++++++----- game-api/src/index.js | 14 ++++++ game-api/src/libjunkdrawer/archives.js | 6 +-- game-api/src/libjunkdrawer/fsPromises.js | 20 ++++++++ game-api/yarn.lock | 28 +++++++++++ gateway/src/index.js | 22 ++++++++- ui/src/components/ServerCard.js | 14 ++++++ 12 files changed, 212 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 8c8b80a..2541558 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,4 @@ package-lock.json gateway/admins.json /game-api/serviceaccount.json +modpack-base/ diff --git a/game-api/package.json b/game-api/package.json index 71eeace..8b982c2 100644 --- a/game-api/package.json +++ b/game-api/package.json @@ -19,6 +19,7 @@ "express": "^4.17.1", "express-prettify": "^0.1.1", "express-timeout-handler": "^2.2.0", + "fs-extra": "^11.1.0", "gamedig": "^3.0.9", "minimist": "^1.2.2", "modern-rcon": "^1.0.3", diff --git a/game-api/src/games/common-helpers.js b/game-api/src/games/common-helpers.js index f534a85..27fe629 100644 --- a/game-api/src/games/common-helpers.js +++ b/game-api/src/games/common-helpers.js @@ -8,11 +8,17 @@ const srcdsRcon = require("srcds-rcon"); const axios = require("axios"); const Gamedig = require("gamedig"); const docker = new (require("dockerode"))(); +const { readFile, writeFile } = require("../libjunkdrawer/fsPromises"); +const { + generateBackupFilename, + makeBackup, +} = require("../libjunkdrawer/archives"); const { game, gameId, gameDir, + connectUrl, debugLog, rconPort, argv, @@ -229,6 +235,58 @@ async function steamWorkshopGetModSearch(appid, query) { return []; } +class BaseGameManager { + constructor() { + if (!this.prepareModPackTempDir) { + this.getModPack = false; + } + } + getConnectUrl() { + return `steam://connect/${connectUrl}`; + } + + /** + * Gets a cached .7z containing mods, or generates a new one if the files have changed. + * This method requires GameManagers to implement getModPackHash() and prepareModPackTempDir. + * @returns filePath.7z + */ + async getModPack() { + const currentHash = await this.getModPackHash(); + const storedHashFilePath = path.join(gameDir, "modPackData.json"); + + let lastModPackData = {}; + try { + lastModPackData = JSON.parse(await readFile(storedHashFilePath)); + } catch (err) {} + if (currentHash === lastModPackData.hash) { + return lastModPackData.filePath; + } + + const modPackFilePath = await this.#buildModPack(); + void writeFile( + storedHashFilePath, + JSON.stringify({ hash: currentHash, filePath: modPackFilePath }) + ); + + return modPackFilePath; + } + + /** + * Creates a new .7z containing mods, to be distributed to clients. + * @returns filePath.7z + */ + async #buildModPack() { + const modPackFilePath = generateBackupFilename(`${gameId}-mods`, gameDir); + const modPackFilename = path.basename(modPackFilePath); + const packTempPath = `/tmp/modpacks/${modPackFilename}`; + + const archiveRootPath = await this.prepareModPackTempDir(packTempPath); + await makeBackup(modPackFilePath, gameDir, [archiveRootPath]); + + return modPackFilePath; + } +} + module.exports = { dockerComposeStart, dockerComposeStop, @@ -243,4 +301,5 @@ module.exports = { readEnvFileCsv, writeEnvFileCsv, steamWorkshopGetModSearch, + BaseGameManager, }; diff --git a/game-api/src/games/docker.js b/game-api/src/games/docker.js index 657344b..eb88fb7 100644 --- a/game-api/src/games/docker.js +++ b/game-api/src/games/docker.js @@ -10,16 +10,15 @@ const { dockerLogRead, gamedigQueryPlayers, rconSRCDSConnect, + BaseGameManager, } = require("./common-helpers"); -module.exports = class GenericDockerManager { +module.exports = class GenericDockerManager extends BaseGameManager { constructor({ getCurrentStatus, setStatus }) { + super(); this.getCurrentStatus = getCurrentStatus; this.setStatus = setStatus; } - getConnectUrl() { - return `steam://connect/${connectUrl}`; - } start() { return dockerComposeStart(); } diff --git a/game-api/src/games/test.js b/game-api/src/games/test.js index c69df68..ab33f50 100644 --- a/game-api/src/games/test.js +++ b/game-api/src/games/test.js @@ -69,4 +69,14 @@ module.exports = class TestManager { filesToBackup() { return ["file1.txt", "dir1"]; } + + getLinks() { + return [ + { + link: "https://store.steampowered.com/app/4000/Garrys_Mod/", + title: "Store", + }, + ]; + return []; + } }; diff --git a/game-api/src/games/valheim.js b/game-api/src/games/valheim.js index 804dc27..aa7c5a5 100644 --- a/game-api/src/games/valheim.js +++ b/game-api/src/games/valheim.js @@ -1,8 +1,18 @@ const Gamedig = require("gamedig"); +const path = require("path"); -const { game, gameId, debugLog, connectUrl, rconPort } = require("../cliArgs"); +const { + game, + gameId, + debugLog, + connectUrl, + rconPort, + gameDir, +} = require("../cliArgs"); const { dockerComposePull, gamedigQueryPlayers } = require("./common-helpers"); const GenericDockerManager = require("./docker"); +const { spawnProcess } = require("../libjunkdrawer/fsPromises"); +const fse = require("fs-extra"); module.exports = class ValheimManager extends GenericDockerManager { getConnectUrl() { @@ -28,15 +38,34 @@ module.exports = class ValheimManager extends GenericDockerManager { async filesToBackup() { return ["saves"]; } - getLinks() { - if (gameId === "valheim") { - return [ - { - link: "https://gman.nebtown.info/files/valheim-mistlands-mods-2022-12-10.7z", - title: "Mod Pack", - }, - ]; - } - return []; + + /** + * Copies mod files to be saved in the modPack.7z into a temporary directory. + * Valheim's assumes the presense of a gmanman/game-setups/valheim/modpack-base/ containing the latest Windows binaries for BepInEx + */ + async prepareModPackTempDir(packTempPath) { + const archiveRootPath = `${packTempPath}/Valheim`; + + await fse.copy( + path.join(__dirname, `../../../game-setups/${game}/modpack-base`), + archiveRootPath + ); + await fse.copy( + path.join(gameDir, "server/BepInEx/config"), + path.join(archiveRootPath, "BepInEx/config") + ); + await fse.copy( + path.join(gameDir, "server/BepInEx/plugins"), + path.join(archiveRootPath, "BepInEx/plugins") + ); + return archiveRootPath; + } + async getModPackHash() { + return ( + await spawnProcess("bash", [ + "-c", + `cd ${gameDir} && find server/BepInEx/{config,plugins} -type f -exec md5sum {} \\; | sort -k 2 | md5sum | cut -d ' ' -f1`, + ]) + ).trim(); } }; diff --git a/game-api/src/index.js b/game-api/src/index.js index bce1aa8..283e77c 100644 --- a/game-api/src/index.js +++ b/game-api/src/index.js @@ -231,6 +231,19 @@ app.get("/mods/search", async (request, response) => { } } }); +app.get("/mods/pack", async (request, response) => { + if (!gameManager.getModPack) { + response.status(501).json({ error: "Not Implemented" }); + } + + try { + const modPackFilePath = await gameManager.getModPack(); + response.download(modPackFilePath); + } catch (err) { + console.error("/mods/pack Error", err); + response.status(500).json({ error: "Failed to build mod pack" }); + } +}); app.post("/rcon", async (request, response) => { if (!gameManager.rcon) { @@ -259,6 +272,7 @@ async function registerWithGateway() { gameManager.getMods && "mods", gameManager.getModList && "modList", gameManager.getModSearch && "modSearch", + gameManager.getModPack && "modPack", gameManager.update && "update", gameManager.filesToBackup && "backup", gameManager.rcon && "rcon", diff --git a/game-api/src/libjunkdrawer/archives.js b/game-api/src/libjunkdrawer/archives.js index 98ce464..e733c7f 100644 --- a/game-api/src/libjunkdrawer/archives.js +++ b/game-api/src/libjunkdrawer/archives.js @@ -2,7 +2,7 @@ const sevenBin = require("7zip-bin"); const sevenzip = require("node-7z"); const path = require("path"); const fsPromises = require("./fsPromises"); -const formatISO = require("date-fns/formatISO"); +const format = require("date-fns/format"); async function makeBackupsDir(gameDir) { const backupsDir = path.join(gameDir, "backups"); @@ -13,9 +13,7 @@ async function makeBackupsDir(gameDir) { } function generateBackupFilename(gameId, gameDir) { - const dateString = formatISO(new Date(), { - format: "basic", - }); + const dateString = format(new Date(), "yyyyMMdd'T'HHmmss"); return path.resolve(gameDir, "backups", `${gameId}-${dateString}.7z`); } diff --git a/game-api/src/libjunkdrawer/fsPromises.js b/game-api/src/libjunkdrawer/fsPromises.js index 88dbfd3..7041b7f 100644 --- a/game-api/src/libjunkdrawer/fsPromises.js +++ b/game-api/src/libjunkdrawer/fsPromises.js @@ -1,5 +1,6 @@ const fs = require("fs"); const util = require("util"); +const child_process = require("child_process"); module.exports = { readFile: util.promisify(fs.readFile), @@ -18,3 +19,22 @@ module.exports.exists = async (fileName) => { return false; } }; + +module.exports.spawnProcess = (cmd, args) => + new Promise((resolve, reject) => { + const cp = child_process.spawn(cmd, args); + const error = []; + const stdout = []; + cp.stdout.on("data", (data) => { + stdout.push(data.toString()); + }); + + cp.on("error", (e) => { + error.push(e.toString()); + }); + + cp.on("close", () => { + if (error.length) reject(error.join("")); + else resolve(stdout.join("")); + }); + }); diff --git a/game-api/yarn.lock b/game-api/yarn.lock index cc255f3..178502e 100644 --- a/game-api/yarn.lock +++ b/game-api/yarn.lock @@ -1097,6 +1097,15 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== +fs-extra@^11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.0.tgz#5784b102104433bb0e090f48bfc4a30742c357ed" + integrity sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -1239,6 +1248,11 @@ graceful-fs@^4.1.2: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.3.tgz#4a12ff1b60376ef09862c2093edd908328be8423" integrity sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ== +graceful-fs@^4.1.6, graceful-fs@^4.2.0: + version "4.2.10" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" + integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== + "graceful-readlink@>= 1.0.0": version "1.0.1" resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" @@ -1503,6 +1517,15 @@ json-stable-stringify-without-jsonify@^1.0.1: resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE= +jsonfile@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" + integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== + dependencies: + universalify "^2.0.0" + optionalDependencies: + graceful-fs "^4.1.6" + jwa@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.4.1.tgz#743c32985cb9e98655530d53641b66c8645b039a" @@ -2442,6 +2465,11 @@ unique-string@^2.0.0: dependencies: crypto-random-string "^2.0.0" +universalify@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" + integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== + unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" diff --git a/gateway/src/index.js b/gateway/src/index.js index 852bc7d..6289ace 100644 --- a/gateway/src/index.js +++ b/gateway/src/index.js @@ -113,14 +113,32 @@ app.all("/:gameId/*", async (request, response) => { const queryParamString = Object.keys(queryParams).length ? "?" + querystring.stringify(queryParams) : ""; - const { data } = await axios({ + const extraAxiosOptions = {}; + if (endpoint.startsWith("mods/pack")) { + extraAxiosOptions.responseType = "stream"; + extraAxiosOptions.decompress = false; + } + const { data, headers: responseHeaders } = await axios({ method: request.method, url: `${requestURL}${queryParamString}`, data: bodyParams, headers, timeout: 5000, + ...extraAxiosOptions, }); - response.json(data); + + if (extraAxiosOptions.responseType === "stream") { + for (const headerName of [ + "content-disposition", + "content-type", + "content-length", + ]) { + response.setHeader(headerName, responseHeaders[headerName]); + } + data.pipe(response); // for file transfers, like modPacks + } else { + response.json(data); + } gameApi.timeoutStartTime = 0; } catch (err) { if (err.code === "ECONNREFUSED") { diff --git a/ui/src/components/ServerCard.js b/ui/src/components/ServerCard.js index d3cac04..7e4be0a 100644 --- a/ui/src/components/ServerCard.js +++ b/ui/src/components/ServerCard.js @@ -252,6 +252,20 @@ export default function ServerCard({ )} + {features.includes("modPack") && ( + + + + )} {links.length > 0 && links.map(({ link, title }, i) => (