diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..38f186e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: Release + +on: + create: + # GitHub Action will not get triggered for '*', so use 'v*' instead + tags: + - v* + # Only used for testing + # push: + # branches-ignore: [ master ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - name: Test + run: make test + - name: Set environment variables + run: echo "CURRENT_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + # run: |- + # echo "WARNING: using test tag" + # echo "CURRENT_TAG=v0.8.5" >> $GITHUB_ENV + - name: Build + run: ./scripts/build.py + - name: Create release and upload assets + run: ./scripts/upload.py $(cat targets.txt) + env: + REPO_OWNER: maximumadmin + REPO_NAME: zramd + GH_RELEASE_TOKEN: ${{ secrets.GH_RELEASE_TOKEN }} diff --git a/.gitignore b/.gitignore index c28751e..740aa5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ +.python/ .vscode/ -zramd.bin +dist/ +*.http diff --git a/Makefile b/Makefile index 0769099..a3f3595 100644 --- a/Makefile +++ b/Makefile @@ -1,37 +1,56 @@ SHELL := /bin/bash MODULE := $(shell sed -nr 's/^module ([a-z\-]+)$$/\1/p' go.mod) GO_FILE := src/$(MODULE).go -OUT_FILE := $(MODULE).bin +ifeq ($(output),) +OUT_FILE := dist/$(MODULE).bin +else +OUT_FILE := $(output) +endif default: - @go version + @{\ + set -e ;\ + os_release_id=$$(grep -E '^ID=' /etc/os-release | sed 's/ID=//' || true) ;\ + if [ "$$os_release_id" = "arch" ]; then \ + make --no-print-directory release-dynamic ;\ + else \ + make --no-print-directory release-static ;\ + fi ;\ + } start: go run $(GO_FILE) clean: go clean - rm -f $(OUT_FILE) + rm -rf dist/* + rm -f "$(OUT_FILE)" -# Build development binary. +# Build development binary build: go build -v -o $(OUT_FILE) $(GO_FILE) - @ls -lh $(OUT_FILE) + @ls -lh "$(OUT_FILE)" -# Build statically linked production binary. -release: clean +# Build statically linked production binary +release-static: + @echo "Building static binary (GOARCH: $(GOARCH) GOARM: $(GOARM))..." @{\ + set -e ;\ + if [ -z "$${skip_clean}" ]; then make --no-print-directory clean; fi ;\ export GOFLAGS="-a -trimpath -ldflags=-w -ldflags=-s" ;\ if [ "$${GOARCH}" != "arm" ]; then \ export GOFLAGS="$${GOFLAGS} -buildmode=pie" ;\ fi ;\ - CGO_ENABLED=0 go build -o $(OUT_FILE) $(GO_FILE) ;\ + CGO_ENABLED=0 go build -o "$(OUT_FILE)" $(GO_FILE) ;\ } - @ls -lh $(OUT_FILE) + @make --no-print-directory postbuild -# Build dinamically linked production binary. -release-dynamic: clean +# Build dinamically linked production binary +release-dynamic: + @echo "Building dynamic binary (GOARCH: $(GOARCH) GOARM: $(GOARM))..." @{\ + set -e ;\ + if [ -z "$${skip_clean}" ]; then make --no-print-directory clean; fi ;\ export CGO_CPPFLAGS="$${CPPFLAGS}" ;\ export CGO_CFLAGS="$${CFLAGS}" ;\ export CGO_CXXFLAGS="$${CXXFLAGS}" ;\ @@ -40,22 +59,50 @@ release-dynamic: clean if [ "$${GOARCH}" != "arm" ]; then \ export GOFLAGS="$${GOFLAGS} -buildmode=pie" ;\ fi ;\ - go build -o $(OUT_FILE) $(GO_FILE) ;\ + go build -o "$(OUT_FILE)" $(GO_FILE) ;\ } - @ls -lh $(OUT_FILE) + @make --no-print-directory postbuild -# Run unit tests on all packages. +postbuild: + @{\ + set -e ;\ + if [ ! -z "$${make_tgz}" ]; then \ + tgz_file="$(OUT_FILE).tar.gz" ;\ + echo "Creating \"$${tgz_file}\"..." ;\ + tar -C "$$(dirname "$(OUT_FILE)")" \ + -cz -f "$$tgz_file" \ + "$$(basename "$(OUT_FILE)")" ;\ + fi ;\ + if [ ! -z "$${make_deb}" ]; then \ + echo "Creating deb ($${DEB_ARCH}) file..." ;\ + CONFIG_FILE=extra/debian.yml \ + ARCH=$${DEB_ARCH} \ + PREFIX="$${PREFIX}" \ + BIN_FILE="$(OUT_FILE)" \ + VERSION=$${VERSION} \ + RELEASE=$${RELEASE} \ + ./scripts/mkdeb.py ;\ + rm -rf "$${PREFIX}" ;\ + fi ;\ + } + @ls -lh "$(OUT_FILE)"* + +# Run unit tests on all packages test: go test -v ./src/... install: - install -Dm755 $(OUT_FILE) "$(PREFIX)/usr/bin/$(MODULE)" + install -Dm755 "$(OUT_FILE)" "$(PREFIX)/usr/bin/$(MODULE)" install -Dm644 LICENSE -t "$(PREFIX)/usr/share/licenses/$(MODULE)/" install -Dm644 extra/$(MODULE).default "$(PREFIX)/etc/default/$(MODULE)" install -Dm644 extra/$(MODULE).service -t "$(PREFIX)/usr/lib/systemd/system/" uninstall: - systemctl disable --now $(MODULE).service + @{\ + if [ -f "$(PREFIX)/usr/lib/systemd/system/$(MODULE).service" ]; then \ + systemctl disable --now $(MODULE).service ;\ + fi ;\ + } rm -f "$(PREFIX)/usr/lib/systemd/system/$(MODULE).service" rm -f "$(PREFIX)/etc/default/$(MODULE)" rm -rf "$(PREFIX)/usr/share/licenses/$(MODULE)/" diff --git a/README.md b/README.md index 5264f0d..85ba0e1 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ See also https://fedoraproject.org/wiki/Changes/SwapOnZRAM#Benefit_to_Fedora * Install `go`, this depends on the distribution you are using e.g. for Ubuntu the command should be `sudo apt-get install golang`. * Run `make release` to make a x86_64 build, to make an ARM build (i.e. for the Raspberry Pi) run `GOOS=linux GOARCH=arm GOARM=7 make release` -* A new executable called `zramd.bin` will be created in the current directory, now you can uninstall `go` if you like. +* A new executable called `zramd.bin` will be created under the `dist/` directory, now you can uninstall `go` if you like. * Optionally on distributions using systemd, you can install `zramd` by just running `make install`, see below for additional installation methods. ## Installation diff --git a/cspell.json b/cspell.json index 0599a4c..006bf7d 100644 --- a/cspell.json +++ b/cspell.json @@ -2,6 +2,7 @@ "version": "0.1", "words": [ "deinitialize", + "itertools", "lsmod", "mkswap", "zram", diff --git a/extra/debian.yml b/extra/debian.yml new file mode 100644 index 0000000..b6fd375 --- /dev/null +++ b/extra/debian.yml @@ -0,0 +1,41 @@ +control: + Package: zramd + Version: ${VERSION}-${RELEASE} + Architecture: ${ARCH} + Maintainer: maximumadmin + Priority: extra + Section: admin + Installed-Size: ${SIZE_KB} + Depends: util-linux + Suggests: earlyoom + Description: Automatically setup swap on zram ✨ + +# https://wiki.debian.org/MaintainerScripts +scripts: + postinst: |- + #!/bin/sh + if [ -d /run/systemd/system ]; then + deb-systemd-invoke enable --now zramd.service >/dev/null || true + fi + prerm: |- + #!/bin/sh + if [ -d /run/systemd/system ] && [ "$1" = remove ]; then + deb-systemd-invoke disable --now zramd.service >/dev/null || true + fi + postrm: |- + if [ -d /run/systemd/system ] && [ "$1" = remove ]; then + systemctl --system daemon-reload >/dev/null || true + fi + +build: + install: + cmd: [make, install] + env: + PREFIX: ${PREFIX} + output: ${BIN_FILE} + # Additional arguments passed to dpkg-deb, see also + # https://manpages.debian.org/jessie/dpkg/dpkg-deb.1.en.html + args: [-Zgzip, -z9] + # Used to rename the final deb file so it does not end up with the same name + # as the root directory (PREFIX), env variables can be used here + rename: zramd_${ARCH}.deb diff --git a/scripts/build.py b/scripts/build.py new file mode 100755 index 0000000..c753b13 --- /dev/null +++ b/scripts/build.py @@ -0,0 +1,64 @@ +#!/usr/bin/python3 -u + +import os +import subprocess +import sys +from typing import Optional, Tuple + +TARGETS = ( + ('arm', '6', 'armel'), + ('arm', '7', 'armhf'), + ('arm64', None, 'arm64'), + ('amd64', None, 'amd64'), +) + +# Parse tag names like v0.8.5 or v0.8.5-1 +def parse_tag(tag: str) -> Tuple[str, str]: + version, release, *_ = [*tag.split('-'), ''] + # Remove the leading 'v' from version + return (version[1:], release or '1') + +def build(goarch: str, goarm: Optional[str], friendly_arch: str) -> int: + out_file = f"dist/zramd_{friendly_arch}" + prefix = f"dist/zramd_{friendly_arch}_root" + version, release = parse_tag(os.environ['CURRENT_TAG']) + proc = subprocess.run( + ['make', f"output={out_file}", 'make_tgz=1', 'make_deb=1', 'skip_clean=1'], + env={ + # Pass all environment variables, contains some Go variables + **os.environ, + # Set Go build-specific variables + 'GOOS': 'linux', + 'GOARCH': goarch, + **({'GOARM': goarm} if goarch == 'arm' else {}), + # Required to create a Debian package + 'DEB_ARCH': friendly_arch, + 'VERSION': version, + 'RELEASE': release, + 'PREFIX': prefix, + 'BIN_FILE': out_file + } + ) + return proc.returncode + +def clean() -> int: + return subprocess.run(['make', 'clean'], env=os.environ).returncode + +def main() -> int: + if (ret := clean()) != 0: + return ret + + # Build all targets sequentially, building in parallel will have minimal or no + # benefit and would make logging messy + for target in TARGETS: + if (ret := build(*target)) != 0: + return ret + + # Finally write the used architectures so we can use them at later steps + with open('targets.txt', 'w') as f: + f.write(','.join(row[2] for row in TARGETS)) + + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/mkdeb.py b/scripts/mkdeb.py new file mode 100755 index 0000000..c6b1b6a --- /dev/null +++ b/scripts/mkdeb.py @@ -0,0 +1,133 @@ +#!/usr/bin/python3 -u + +import os +import pathlib +import re +import subprocess +import sys +import yaml +from typing import List + +# The outer capture group will grab the whole variable including the dollar sign +# and brackets, the inner capture group will only grab the variable name itself +# e.g. '${VER}-${REL}' -> [('${VER}', 'VER'), ('${REL}', 'REL')] +ENV_RE = re.compile(r'(\$\{([A-Za-z0-9\_]+)\})') + +def print_error(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +def read_config(file: str) -> str: + with open(file, 'r') as f: + return yaml.safe_load(f) + +# Get the total size of a directory in bytes +def dir_size(path: str) -> int: + root = pathlib.Path(path) + return sum(f.stat().st_size for f in root.glob('**/*') if f.is_file()) + +# Parse strings containing env variables and make replacements if applicable +# e.g. '${VER}-${REL}' -> '0.8.4-1' +def parse_env(text: str, env: dict) -> str: + result = text + for expr, name in ENV_RE.findall(text): + if (env_val := env.get(name)) is not None: + result = result.replace(expr, env_val) + return result + +def write_control_file(prefix: str, data: dict, env: dict) -> None: + lines = '' + for key, val in data.items(): + lines += f"{key}: {parse_env(val, env)}\n" + with open(os.path.join(prefix, 'DEBIAN/control'), 'w+') as f: + f.write(lines) + +def write_script(prefix: str, filename: str, content: str) -> None: + script = os.path.join(prefix, f"DEBIAN/{filename}") + with open(script, 'w+') as f: + f.write(content) + os.chmod(script, 0o775) + +def write_conffiles(prefix: str) -> None: + root = pathlib.Path(os.path.join(prefix, 'etc')) + files = ( + '/' + os.path.relpath(str(f), prefix) + for f in root.glob('**/*') + if f.is_file() + ) + # As per dpkg-deb requirement we need a newline at the end of the file + content = '\n'.join(files) + '\n' + with open(os.path.join(prefix, 'DEBIAN/conffiles'), 'w+') as f: + f.write(content) + +def write_md5sums(prefix: str) -> None: + cmd = r""" + find . -mindepth 1 -type f -not -path './DEBIAN/*' |\ + sed 's|^./||' | sort | xargs md5sum + """ + # https://docs.python.org/3/library/subprocess.html#subprocess.check_output + output = subprocess.check_output( + cmd, + shell=True, + cwd=prefix, + text=True + ).strip() + with open(os.path.join(prefix, 'DEBIAN/md5sums'), 'w+') as f: + f.write(output) + +def make_deb(prefix: str, args: List[str]) -> int: + final_args = ['dpkg-deb', *args, '--build', prefix] + return subprocess.run(final_args).returncode + +def main() -> int: + if not (config_file := os.environ.get('CONFIG_FILE')): + print_error('the CONFIG_FILE variable is not set') + return 1 + if not (prefix := os.environ.get('PREFIX')): + print_error('the PREFIX variable is not set') + return 1 + + pathlib.Path(os.path.join(prefix, 'DEBIAN')).mkdir( + parents=True, + exist_ok=True + ) + + config: dict = read_config(config_file) + + install_cmd = ( + cmd + if (cmd := config.get('build', {}).get('install', {}).get('cmd')) + else ['make', 'install'] + ) + install_env = config.get('build', {}).get('install', {}).get('env', {}) + for key, val in install_env.items(): + install_env[key] = parse_env(val, os.environ) + if (ret := subprocess.run(install_cmd, env=install_env).returncode) != 0: + return ret + + env = { + **os.environ, + 'SIZE_KB': os.environ.get('SIZE_KB') or str(int(dir_size(prefix) / 1024)) + } + write_control_file(prefix, config.get('control', {}), env) + + scripts = config.get('scripts', {}) + for filename, content in scripts.items(): + write_script(prefix, filename, content) + + write_conffiles(prefix) + + write_md5sums(prefix) + + args = config.get('build', {}).get('args', []) + if (ret := make_deb(prefix, args)) != 0: + return ret + + if (target_name := config.get('build', {}).get('rename')): + dir_name = os.path.dirname(prefix) + final_name = parse_env(target_name, env) + os.rename(f"{prefix}.deb", os.path.join(dir_name, final_name)) + + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/upload.py b/scripts/upload.py new file mode 100755 index 0000000..11c681d --- /dev/null +++ b/scripts/upload.py @@ -0,0 +1,161 @@ +#!/usr/bin/python3 -u + +import itertools +import json +import os +import subprocess +import sys +import time +import urllib.parse +from http.client import HTTPResponse +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen +from typing import Iterable, List, Optional, Tuple + +Response = Tuple[Optional[Exception], Optional[dict]] +Content = Optional[object] + +MAX_RETRIES = 2 +DEFAULT_HEADERS = { + 'Content-Type': 'application/json', + 'Accept': 'application/vnd.github.v3+json' +} + +def print_error(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +def flatten(iterable: Iterable) -> List: + return list(itertools.chain(*iterable)) + +def request_json(url: str, headers: Optional[dict], body=None) -> Response: + req = Request(url, method='POST') + for key, value in {**DEFAULT_HEADERS, **(headers or {})}.items(): + req.add_header(key, value) + body_bytes = json.dumps(body).encode('utf-8') if body else None + response_body: bytes + response_code: int + try: + res: HTTPResponse + with urlopen(req, data=body_bytes) as res: + response_body = res.read() + response_code = res.getcode() + except (HTTPError, URLError) as e: + return (e, None) + response_json: dict = json.loads(response_body) + if response_code >= 400: + message = {'status': response_code, 'body': response_body} + return (Exception(json.dumps(message, indent=2)), None) + return (None, response_json) + +def request_safe( + url: str, + headers: Optional[dict], + body=None, + max_retries=MAX_RETRIES +) -> Content: + i = 0 + while True: + url_path = urllib.parse.urlparse(url).path + print(f"[{i}] Connecting to \"{url_path}\"...") + (e, data) = request_json(url, headers, body) + if not e: + return data + print_error(e) + if max_retries > 0 and (i == max_retries - 1): + break + time.sleep(2) + i += 1 + return None + +def create_release(owner: str, repo: str, token: str, body: dict) -> Content: + url = f"https://api.github.com/repos/{owner}/{repo}/releases" + headers = {'Authorization': f"token {token}"} + return request_safe(url, headers, body) + +# Easier than do it in pure Python and does not require additional libraries +def curl_upload( + upload_url: str, + headers: dict, + file_path: str +) -> Tuple[int, int]: + header_list = flatten(['-H', f"{h}: {headers[h]}"] for h in headers) + curl_args = [ + # Be silent except on errors + '-sS', + # Hide the response output (may contain hidden URLs and will pollute stdout) + '-o', '/dev/null', + # Write response code to stdout + '-w', '%{http_code}', + # Spread header list + *header_list, + # Path to the actual binary + '--data-binary', f"@{file_path}" + ] + result = subprocess.run( + ['curl', *curl_args, upload_url], + stdout=subprocess.PIPE, + text=True + ) + status = int(result.stdout) if result.returncode == 0 else 0 + return (result.returncode, status) + +def upload_asset( + upload_url: str, + token: str, + file_path: str, + max_retries=MAX_RETRIES +) -> bool: + headers = { + 'Authorization': f"token {token}", + 'Content-Type': 'application/octet-stream' + } + for i in range(max_retries): + print(f"[{i}] Uploading \"{file_path}\"...") + ret, status = curl_upload(upload_url, headers, file_path) + if ret == 0 and (200 <= status < 400): + break + print_error(f"curl finished with exit code {ret}") + if i == max_retries - 1: + return False + return True + +def main() -> int: + # Get all required variables from env, if we get an error that's good, it just + # means that we forgot to pass a variable + owner = os.environ['REPO_OWNER'] + repo = os.environ['REPO_NAME'] + token = os.environ['GH_RELEASE_TOKEN'] + release_tag = os.environ['CURRENT_TAG'] + + # Get list of friendly architectures from the first argument, each item will + # have a corresponding binary, tar and deb file under dist/ + friendly_arches = sys.argv[1].split(',') + + # Create a GitHub release (requires a valid tag to exist) + release_data = create_release(owner, repo, token, { + 'tag_name': release_tag, + 'name': release_tag, + 'body': f"zramd {release_tag}" + }) + if not release_data: + return 1 + + # https://uploads.github.com/repos/OWNER/REPO/releases/ID/assets{?name,label} + upload_url: str = release_data['upload_url'] + upload_url = upload_url.split('/assets')[0] + '/assets?name=' + + # Upload assets + assets = flatten( + (f"zramd_{a}.deb", f"zramd_{a}.tar.gz") + for a in friendly_arches + ) + for asset in assets: + current_file = os.path.join('dist', asset) + current_url = upload_url + os.path.basename(current_file) + if not upload_asset(current_url, token, current_file): + return 1 + + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/utsname/utsname_arm64.go b/src/utsname/utsname_arm64.go new file mode 100644 index 0000000..7110dd2 --- /dev/null +++ b/src/utsname/utsname_arm64.go @@ -0,0 +1,12 @@ +package utsname + +func parseCharSlice(data []int8) string { + b := make([]byte, 0, len(data)) + for _, v := range data { + if v == 0x00 { + break + } + b = append(b, byte(v)) + } + return string(b) +}