New Feature: Mod Pack 7z generation + hosting (Valheim)

.gitignore

@@ -80,3 +80,4 @@ package-lock.json
 
 gateway/admins.json
 /game-api/serviceaccount.json
+modpack-base/

game-api/package.json

@@ -19,6 +19,7 @@
     "express": "^4.17.1",
     "express-prettify": "^0.1.1",
     "express-timeout-handler": "^2.2.0",
+    "fs-extra": "^11.1.0",
     "gamedig": "^3.0.9",
     "minimist": "^1.2.2",
     "modern-rcon": "^1.0.3",

game-api/src/games/common-helpers.js

@@ -8,11 +8,17 @@ const srcdsRcon = require("srcds-rcon");
 const axios = require("axios");
 const Gamedig = require("gamedig");
 const docker = new (require("dockerode"))();
+const { readFile, writeFile } = require("../libjunkdrawer/fsPromises");
+const {
+	generateBackupFilename,
+	makeBackup,
+} = require("../libjunkdrawer/archives");
 
 const {
 	game,
 	gameId,
 	gameDir,
+	connectUrl,
 	debugLog,
 	rconPort,
 	argv,
@@ -229,6 +235,58 @@ async function steamWorkshopGetModSearch(appid, query) {
 	return [];
 }
 
+class BaseGameManager {
+	constructor() {
+		if (!this.prepareModPackTempDir) {
+			this.getModPack = false;
+		}
+	}
+	getConnectUrl() {
+		return `steam://connect/${connectUrl}`;
+	}
+
+	/**
+	 * Gets a cached .7z containing mods, or generates a new one if the files have changed.
+	 * This method requires GameManagers to implement getModPackHash() and prepareModPackTempDir.
+	 * @returns filePath.7z
+	 */
+	async getModPack() {
+		const currentHash = await this.getModPackHash();
+		const storedHashFilePath = path.join(gameDir, "modPackData.json");
+
+		let lastModPackData = {};
+		try {
+			lastModPackData = JSON.parse(await readFile(storedHashFilePath));
+		} catch (err) {}
+		if (currentHash === lastModPackData.hash) {
+			return lastModPackData.filePath;
+		}
+
+		const modPackFilePath = await this.#buildModPack();
+		void writeFile(
+			storedHashFilePath,
+			JSON.stringify({ hash: currentHash, filePath: modPackFilePath })
+		);
+
+		return modPackFilePath;
+	}
+
+	/**
+	 * Creates a new .7z containing mods, to be distributed to clients.
+	 * @returns filePath.7z
+	 */
+	async #buildModPack() {
+		const modPackFilePath = generateBackupFilename(`${gameId}-mods`, gameDir);
+		const modPackFilename = path.basename(modPackFilePath);
+		const packTempPath = `/tmp/modpacks/${modPackFilename}`;
+
+		const archiveRootPath = await this.prepareModPackTempDir(packTempPath);
+		await makeBackup(modPackFilePath, gameDir, [archiveRootPath]);
+
+		return modPackFilePath;
+	}
+}
+
 module.exports = {
 	dockerComposeStart,
 	dockerComposeStop,
@@ -243,4 +301,5 @@ module.exports = {
 	readEnvFileCsv,
 	writeEnvFileCsv,
 	steamWorkshopGetModSearch,
+	BaseGameManager,
 };

game-api/src/games/docker.js

@@ -10,16 +10,15 @@ const {
 	dockerLogRead,
 	gamedigQueryPlayers,
 	rconSRCDSConnect,
+	BaseGameManager,
 } = require("./common-helpers");
 
-module.exports = class GenericDockerManager {
+module.exports = class GenericDockerManager extends BaseGameManager {
 	constructor({ getCurrentStatus, setStatus }) {
+		super();
 		this.getCurrentStatus = getCurrentStatus;
 		this.setStatus = setStatus;
 	}
-	getConnectUrl() {
-		return `steam://connect/${connectUrl}`;
-	}
 	start() {
 		return dockerComposeStart();
 	}

game-api/src/games/test.js

@@ -69,4 +69,14 @@ module.exports = class TestManager {
 	filesToBackup() {
 		return ["file1.txt", "dir1"];
 	}
+
+	getLinks() {
+		return [
+			{
+				link: "https://store.steampowered.com/app/4000/Garrys_Mod/",
+				title: "Store",
+			},
+		];
+		return [];
+	}
 };

game-api/src/games/valheim.js

@@ -1,8 +1,18 @@
 const Gamedig = require("gamedig");
+const path = require("path");
 
-const { game, gameId, debugLog, connectUrl, rconPort } = require("../cliArgs");
+const {
+	game,
+	gameId,
+	debugLog,
+	connectUrl,
+	rconPort,
+	gameDir,
+} = require("../cliArgs");
 const { dockerComposePull, gamedigQueryPlayers } = require("./common-helpers");
 const GenericDockerManager = require("./docker");
+const { spawnProcess } = require("../libjunkdrawer/fsPromises");
+const fse = require("fs-extra");
 
 module.exports = class ValheimManager extends GenericDockerManager {
 	getConnectUrl() {
@@ -28,15 +38,34 @@ module.exports = class ValheimManager extends GenericDockerManager {
 	async filesToBackup() {
 		return ["saves"];
 	}
-	getLinks() {
-		if (gameId === "valheim") {
-			return [
-				{
-					link: "https://gman.nebtown.info/files/valheim-mistlands-mods-2022-12-10.7z",
-					title: "Mod Pack",
-				},
-			];
-		}
-		return [];
+
+	/**
+	 * Copies mod files to be saved in the modPack.7z into a temporary directory.
+	 * Valheim's assumes the presense of a gmanman/game-setups/valheim/modpack-base/ containing the latest Windows binaries for BepInEx
+	 */
+	async prepareModPackTempDir(packTempPath) {
+		const archiveRootPath = `${packTempPath}/Valheim`;
+
+		await fse.copy(
+			path.join(__dirname, `../../../game-setups/${game}/modpack-base`),
+			archiveRootPath
+		);
+		await fse.copy(
+			path.join(gameDir, "server/BepInEx/config"),
+			path.join(archiveRootPath, "BepInEx/config")
+		);
+		await fse.copy(
+			path.join(gameDir, "server/BepInEx/plugins"),
+			path.join(archiveRootPath, "BepInEx/plugins")
+		);
+		return archiveRootPath;
+	}
+	async getModPackHash() {
+		return (
+			await spawnProcess("bash", [
+				"-c",
+				`cd ${gameDir} && find server/BepInEx/{config,plugins} -type f -exec md5sum {} \\; | sort -k 2 | md5sum | cut -d ' ' -f1`,
+			])
+		).trim();
 	}
 };

game-api/src/index.js

@@ -231,6 +231,19 @@ app.get("/mods/search", async (request, response) => {
 		}
 	}
 });
+app.get("/mods/pack", async (request, response) => {
+	if (!gameManager.getModPack) {
+		response.status(501).json({ error: "Not Implemented" });
+	}
+
+	try {
+		const modPackFilePath = await gameManager.getModPack();
+		response.download(modPackFilePath);
+	} catch (err) {
+		console.error("/mods/pack Error", err);
+		response.status(500).json({ error: "Failed to build mod pack" });
+	}
+});
 
 app.post("/rcon", async (request, response) => {
 	if (!gameManager.rcon) {
@@ -259,6 +272,7 @@ async function registerWithGateway() {
 				gameManager.getMods && "mods",
 				gameManager.getModList && "modList",
 				gameManager.getModSearch && "modSearch",
+				gameManager.getModPack && "modPack",
 				gameManager.update && "update",
 				gameManager.filesToBackup && "backup",
 				gameManager.rcon && "rcon",

game-api/src/libjunkdrawer/archives.js

@@ -2,7 +2,7 @@ const sevenBin = require("7zip-bin");
 const sevenzip = require("node-7z");
 const path = require("path");
 const fsPromises = require("./fsPromises");
-const formatISO = require("date-fns/formatISO");
+const format = require("date-fns/format");
 
 async function makeBackupsDir(gameDir) {
 	const backupsDir = path.join(gameDir, "backups");
@@ -13,9 +13,7 @@ async function makeBackupsDir(gameDir) {
 }
 
 function generateBackupFilename(gameId, gameDir) {
-	const dateString = formatISO(new Date(), {
-		format: "basic",
-	});
+	const dateString = format(new Date(), "yyyyMMdd'T'HHmmss");
 	return path.resolve(gameDir, "backups", `${gameId}-${dateString}.7z`);
 }
 

game-api/src/libjunkdrawer/fsPromises.js

@@ -1,5 +1,6 @@
 const fs = require("fs");
 const util = require("util");
+const child_process = require("child_process");
 
 module.exports = {
 	readFile: util.promisify(fs.readFile),
@@ -18,3 +19,22 @@ module.exports.exists = async (fileName) => {
 		return false;
 	}
 };
+
+module.exports.spawnProcess = (cmd, args) =>
+	new Promise((resolve, reject) => {
+		const cp = child_process.spawn(cmd, args);
+		const error = [];
+		const stdout = [];
+		cp.stdout.on("data", (data) => {
+			stdout.push(data.toString());
+		});
+
+		cp.on("error", (e) => {
+			error.push(e.toString());
+		});
+
+		cp.on("close", () => {
+			if (error.length) reject(error.join(""));
+			else resolve(stdout.join(""));
+		});
+	});

game-api/yarn.lock

@@ -1097,6 +1097,15 @@ fs-constants@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
   integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==
 
+fs-extra@^11.1.0:
+  version "11.1.0"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.0.tgz#5784b102104433bb0e090f48bfc4a30742c357ed"
+  integrity sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw==
+  dependencies:
+    graceful-fs "^4.2.0"
+    jsonfile "^6.0.1"
+    universalify "^2.0.0"
+
 fs.realpath@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
@@ -1239,6 +1248,11 @@ graceful-fs@^4.1.2:
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.3.tgz#4a12ff1b60376ef09862c2093edd908328be8423"
   integrity sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==
 
+graceful-fs@^4.1.6, graceful-fs@^4.2.0:
+  version "4.2.10"
+  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c"
+  integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==
+
 "graceful-readlink@>= 1.0.0":
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725"
@@ -1503,6 +1517,15 @@ json-stable-stringify-without-jsonify@^1.0.1:
   resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651"
   integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=
 
+jsonfile@^6.0.1:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"
+  integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==
+  dependencies:
+    universalify "^2.0.0"
+  optionalDependencies:
+    graceful-fs "^4.1.6"
+
 jwa@^1.4.1:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.4.1.tgz#743c32985cb9e98655530d53641b66c8645b039a"
@@ -2442,6 +2465,11 @@ unique-string@^2.0.0:
   dependencies:
     crypto-random-string "^2.0.0"
 
+universalify@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717"
+  integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==
+
 [email protected], unpipe@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"

gateway/src/index.js

@@ -113,14 +113,32 @@ app.all("/:gameId/*", async (request, response) => {
 		const queryParamString = Object.keys(queryParams).length
 			? "?" + querystring.stringify(queryParams)
 			: "";
-		const { data } = await axios({
+		const extraAxiosOptions = {};
+		if (endpoint.startsWith("mods/pack")) {
+			extraAxiosOptions.responseType = "stream";
+			extraAxiosOptions.decompress = false;
+		}
+		const { data, headers: responseHeaders } = await axios({
 			method: request.method,
 			url: `${requestURL}${queryParamString}`,
 			data: bodyParams,
 			headers,
 			timeout: 5000,
+			...extraAxiosOptions,
 		});
-		response.json(data);
+
+		if (extraAxiosOptions.responseType === "stream") {
+			for (const headerName of [
+				"content-disposition",
+				"content-type",
+				"content-length",
+			]) {
+				response.setHeader(headerName, responseHeaders[headerName]);
+			}
+			data.pipe(response); // for file transfers, like modPacks
+		} else {
+			response.json(data);
+		}
 		gameApi.timeoutStartTime = 0;
 	} catch (err) {
 		if (err.code === "ECONNREFUSED") {

ui/src/components/ServerCard.js

@@ -252,6 +252,20 @@ export default function ServerCard({
 							)}
 							<Grid item>
 								<Grid container direction="row" spacing={1}>
+									{features.includes("modPack") && (
+										<Grid item xs={6}>
+											<Button
+												size="small"
+												href={`${baseUrl}mods/pack`}
+												variant="outlined"
+												style={{
+													whiteSpace: "nowrap",
+												}}
+											>
+												Mod Pack
+											</Button>
+										</Grid>
+									)}
 									{links.length > 0 &&
 										links.map(({ link, title }, i) => (
 											<Grid item xs={6} key={`link-${id}-${i}`}>