From b08d46a7ffdce19f649da4f8b01cdc2cb3f1aecb Mon Sep 17 00:00:00 2001 From: Jag_k <30597878+jag-k@users.noreply.github.com> Date: Fri, 1 Dec 2023 03:03:46 +0400 Subject: [PATCH] Add linter to PR and minify Docker image (#3) * Add linters for PR * Update Dockerfile and Docker image * Update building process * Update a Python version to 3.12 --- .dockerignore | 6 +- .github/dependabot.yml | 26 +++++++ .github/release.yml | 1 + .github/workflows/deploy.yml | 123 +++++++++++++++++++++++++++------- .github/workflows/pr_lint.yml | 111 ++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 2 +- Dockerfile | 42 +++++++++--- app/__init__.py | 0 app/client.py | 16 +++-- app/config/__init__.py | 9 +-- app/config/base.py | 2 +- app/generate_tfl.py | 14 ++-- app/main.py | 8 +-- poetry.lock | 4 +- pyproject.toml | 69 ++++++++++++------- 15 files changed, 345 insertions(+), 88 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/pr_lint.yml create mode 100644 app/__init__.py diff --git a/.dockerignore b/.dockerignore index 4b5db37..f9a85b8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,7 @@ -.env NPSdir TORRENTdir -.editorconfig -.gitignore app/config/local.py -.dockerignore docker-compose.yml docker-compose.local.yml +wiki/ +.* diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8aeb5a1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,26 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + + assignees: + - "jag-k" + + # Maintain dependencies for Poetry + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + + # Include a list of updated dependencies + # with a prefix determined by the dependency group + commit-message: + prefix: "poetry" + prefix-development: "poetry dev" + include: "scope" + + assignees: + - "jag-k" diff --git a/.github/release.yml b/.github/release.yml index 46adade..55595de 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -4,6 +4,7 @@ changelog: - ignore-for-release authors: - octocat + - dependabot categories: - title: Breaking Changes 🛠 labels: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1cb8c49..2894a66 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -9,64 +9,141 @@ on: branches: [ "main" ] tags: - "v*.*.*" + pull_request: + types: [ closed ] + branches: + - 'release/*.*.*' workflow_dispatch: permissions: contents: write packages: write + pull-requests: write jobs: - build: + lint: runs-on: ubuntu-latest - permissions: - contents: read - packages: write + outputs: + version: ${{ steps.get_version.outputs.version }} + is_pr: ${{ steps.release.outputs.is_pr == 'true'}} + is_tag: ${{ steps.release.outputs.is_tag == 'true'}} + is_release: ${{ steps.release.outputs.is_release == 'true'}} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install poetry + run: pipx install poetry + - uses: actions/setup-python@v4 + with: + python-version: '3.12' + cache: 'poetry' + - name: Install dependencies + run: poetry install + + - name: Get version from pyproject.toml + id: get_version + run: | + echo "version=$(poetry version -s)" >> $GITHUB_OUTPUT + + - name: Is it a releasable? + id: release + run: | + is_pr="${{ github.event_name == 'pull_request' && github.event.pull_request.merged == true }}" + is_tag="${{ startsWith(github.ref, 'refs/tags/') }}" + [[ "$is_pr" == "true" || "$is_tag" == "true" ]] && is_release=true || is_release=false + + echo "is_pr=$is_pr" >> $GITHUB_OUTPUT + echo "is_tag=$is_tag" >> $GITHUB_OUTPUT + echo "is_release=$is_release" >> $GITHUB_OUTPUT + + - name: Run Ruff + run: poetry run ruff check --output-format=github . + + - name: Run pre-commit hooks + run: poetry run pre-commit run --all-files --show-diff-on-failure + + + build: + needs: + - lint + runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Log in to the Container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Get version from pyproject.toml - id: get_version - run: | - echo "version=$(grep -E '^\[tool\.poetry\]$' -A 4 pyproject.toml | grep -E '^version' | sed 's/version = "//;s/"$//')" >> $GITHUB_OUTPUT - - - name: Get short SHA - id: slug - run: echo "sha=$(echo ${GITHUB_SHA} | cut -c1-7)" >> $GITHUB_OUTPUT + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + tags: | + type=ref,event=branch + type=sha + type=semver,pattern={{version}},value=${{ needs.lint.outputs.version }},enabled=${{ needs.lint.outputs.is_release }} + type=semver,pattern={{major}}.{{minor}},value=${{ needs.lint.outputs.version }},enabled=${{ needs.lint.outputs.is_release }} + type=semver,pattern={{major}},value=${{ needs.lint.outputs.version }},enabled=${{ needs.lint.outputs.is_release }} + type=raw,value=latest,enable={{is_default_branch}} - name: Build and push Docker image + id: docker_build uses: docker/build-push-action@v3 with: context: . push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.slug.outputs.sha }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_version.outputs.version }} + tags: ${{ steps.meta.outputs.tags }} cache-from: type=gha cache-to: type=gha,mode=max + + release: + runs-on: ubuntu-latest + if: ${{ needs.lint.outputs.is_release }} + needs: + - build + steps: + - name: Create Tag + id: create_tag + uses: actions/github-script@v7 + if: ${{ needs.lint.outputs.is_pr }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + # language=JavaScript + script: | + const version = "${{ needs.lint.outputs.version }}" + const tagMessage = `Release version ${version}` + github.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/tags/v${version}`, + sha: context.sha + }) + github.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: `v${version}`, + name: `v${version}`, + body: tagMessage + }) + - name: Release uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') + with: body: | - Docker image: `${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_version.outputs.version }}` + Docker image: [`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_version.outputs.version }}`](https://github.com/jag-k/owntinfoil/pkgs/container/owntinfoil/${{ steps.docker_build.outputs }}?tag=${{ steps.get_version.outputs.version }}) tag_name: v${{ steps.get_version.outputs.version }} generate_release_notes: true append_body: true - token: ${{ secrets.RELEASE_TOKEN }} - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} diff --git a/.github/workflows/pr_lint.yml b/.github/workflows/pr_lint.yml new file mode 100644 index 0000000..d4d1d68 --- /dev/null +++ b/.github/workflows/pr_lint.yml @@ -0,0 +1,111 @@ +name: PR Validation + +on: + pull_request: + +jobs: + validate-pr: + runs-on: ubuntu-latest + steps: + - name: Check labels + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + # language=JavaScript + script: | + const { data: pullRequest } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + + const labels = pullRequest.labels.map(label => label.name); + const invalidLabels = ['feature request', 'invalid', 'question']; + const hasInvalidLabel = labels.some(label => invalidLabels.includes(label)); + + if (labels.length === 0) { + core.setFailed('Pull request must have at least one label!'); + } + + if (hasInvalidLabel) { + core.setFailed('Pull request must have an invalid label: ' + invalidLabels.join(', ')); + } + + - name: Check assignees + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + # language=JavaScript + script: | + const { data: pullRequest } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + + if (pullRequest.assignees.length === 0) { + core.setFailed('Pull request must have an assignee'); + } + + - name: Check draft status + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + # language=JavaScript + script: | + const { data: pullRequest } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + + if (pullRequest.draft) { + core.setFailed('Pull request must not be a draft'); + } + + validate-code: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get_version.outputs.version }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install poetry + run: pipx install poetry + - uses: actions/setup-python@v4 + with: + python-version: '3.12' + cache: 'poetry' + + - name: Install dependencies + run: poetry install + + - name: Run Ruff + run: poetry run ruff check --output-format=github . + + - name: Run pre-commit hooks + run: poetry run pre-commit run --all-files --show-diff-on-failure + + - name: Get version from pyproject.toml + id: get_version + run: | + echo "version=$(poetry version -s)" >> $GITHUB_OUTPUT + + validate-branch: + runs-on: ubuntu-latest + needs: + - validate-code + + if: startsWith(github.head_ref, 'release/') + steps: + - name: Check branch name with version + run: | + # Extract the branch name from the GitHub context + branch_name=${{ github.head_ref }} + + # Check if the branch name starts with "release/" and does not match the version + if [[ $branch_name == release/* && $branch_name != "release/${{ needs.validate-code.outputs.version }}" ]]; then + echo "Branch name does not match the version in pyproject.toml" + exit 1 + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 947bf7e..15fe297 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/Dockerfile b/Dockerfile index b769224..f426a4b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,41 @@ -FROM python:3.11-slim +FROM python:3.12-slim as base -LABEL org.opencontainers.image.source = "https://github.com/jag-k/owntinfoil" +LABEL org.opencontainers.image.source="https://github.com/jag-k/owntinfoil" +LABEL org.opencontainers.image.description="OwnTinfoil Image" +LABEL org.opencontainers.image.licenses="MIT" -WORKDIR /app - -ENV NPS_DIR=/nps -ENV TORRENT_DIR=/torrents +ENV PIP_DEFAULT_TIMEOUT=100 \ + # Allow statements and log messages to immediately appear + PYTHONUNBUFFERED=1 \ + # disable a pip version check to reduce run-time & log-spam + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + # cache is useless in docker image, so disable to reduce image size + PIP_NO_CACHE_DIR=1 ENV PYTHONPATH=/app -ENV POETRY_VERSION=1.7.1 +WORKDIR $PYTHONPATH +FROM base as builder + +ARG POETRY_VERSION=1.7.1 RUN pip install "poetry==$POETRY_VERSION" + COPY pyproject.toml poetry.lock ./ -RUN poetry config virtualenvs.create false && \ - poetry install --without dev -COPY . /app +# Set poetry config to install packages in /venv/project/lib/python3.12/site-packages +RUN poetry config virtualenvs.path "/venv" && \ + poetry config virtualenvs.prompt "project" && \ + poetry install --no-root --only main + + +FROM base as final + +ENV NPS_DIR=/nps +ENV TORRENT_DIR=/torrents + +# Copy python side-packages from builder +COPY --from=builder "/venv/project/lib/python3.12/site-packages" "/usr/local/lib/python3.11/site-packages" + +COPY . $PYTHONPATH +#ENTRYPOINT ["/bin/bash"] CMD ["python", "app/main.py"] diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/client.py b/app/client.py index 27019e5..94b44b5 100644 --- a/app/client.py +++ b/app/client.py @@ -4,6 +4,7 @@ import sys import urllib.parse from dataclasses import dataclass +from pathlib import Path from typing import Self import config @@ -11,7 +12,7 @@ from aiohttp import BasicAuth, ClientConnectorError, ClientSession, ServerDisconnectedError -def gen_session(): +def gen_session() -> ClientSession: return ClientSession(auth=BasicAuth("user", config.BOT_KEY)) @@ -19,10 +20,10 @@ class BaseClient: def __init__(self, print_errors: bool = True): self._session = gen_session() self._open_count = 0 - self._open_result = None + self._open_result: ClientSession | None = None self._print_errors = print_errors - async def __aenter__(self): + async def __aenter__(self) -> ClientSession: self._open_count += 1 if self._open_count == 1: self._open_result = await self._session.__aenter__() @@ -74,26 +75,27 @@ async def get_data(self, retries: int = 2) -> bytes: print(f"Server disconnected for file {self.name!r}, retrying...", file=sys.stderr) retries -= 1 print(f"Failed to download file {self.name!r} after {original_retries} retries!", file=sys.stderr) + return None @staticmethod def normalize_name(name: str) -> str: return html.unescape(urllib.parse.unquote(name)).split("_&&_")[-1] @property - def abs_url(self): + def abs_url(self) -> str: return f"{config.BOT_URL.rstrip('/')}/{self.data_url}" - async def save_torrent(self): + async def save_torrent(self) -> Path | None: path = config.TORRENT_DIR / self.name data = await self.get_data() if not data: print(f"Failed to download torrent {self.name!r}! Skipping...", file=sys.stderr) - return + return None async with async_open(path, "wb") as file: await file.write(data) return path - def __str__(self): + def __str__(self) -> str: return f"{self.name} ({self.abs_url})" diff --git a/app/config/__init__.py b/app/config/__init__.py index dd46bb3..5ba8ba1 100644 --- a/app/config/__init__.py +++ b/app/config/__init__.py @@ -1,3 +1,4 @@ +import contextlib import os from pathlib import Path @@ -5,12 +6,9 @@ from app.config.base import * -try: +with contextlib.suppress(ImportError): from app.config.local import * -except ImportError: - pass - load_dotenv() for key in dir(): @@ -37,3 +35,6 @@ HTTP_AUTH_USERS[user] = password except ValueError: continue + +if BOT_KEY is None: + raise ValueError("BOT_KEY must be set!") diff --git a/app/config/base.py b/app/config/base.py index 30396aa..6eacd36 100644 --- a/app/config/base.py +++ b/app/config/base.py @@ -12,7 +12,7 @@ "before tinfoil shops and absolutely free!" ) -HOST: str = "0.0.0.0" +HOST: str = "0.0.0.0" # noqa: S104 PORT: int = 8080 __all__ = ( diff --git a/app/generate_tfl.py b/app/generate_tfl.py index 1558de9..c881764 100644 --- a/app/generate_tfl.py +++ b/app/generate_tfl.py @@ -1,9 +1,10 @@ import os +from pathlib import Path from config import NPS_DIR, SUCCESS_MESSAGE -def _normalize_path(path: str) -> str: +def _normalize_path(path: Path | str) -> str: """Normalize path""" return "../" + str(path).lstrip(".").lstrip("/").lstrip("\\") @@ -13,17 +14,18 @@ def generate_tfl_file(path: str = NPS_DIR, message: str = SUCCESS_MESSAGE) -> di files_result: list[dict] = [] dirs_result: set[str] = set() for _dirname, dirs, files in os.walk(path): - dirname = os.path.relpath(_dirname, path) + _dirname = Path(_dirname) + dirname = _dirname.relative_to(path) for d in dirs: - dirs_result.add(os.path.join(dirname, d)) + dirs_result.add(str(dirname / d)) for f in files: if f.endswith(".part"): continue files_result.append( { - "url": _normalize_path(os.path.join(dirname, f)), - "size": os.path.getsize(os.path.join(_dirname, f)), - } + "url": _normalize_path(dirname / f), + "size": (_dirname / f).stat().st_size, + }, ) return { "files": files_result, diff --git a/app/main.py b/app/main.py index 4bad61c..a879a0b 100644 --- a/app/main.py +++ b/app/main.py @@ -12,15 +12,15 @@ class BasicAuth(BasicAuthMiddleware): - async def check_credentials(self, username: str, password: str, request): + async def check_credentials(self, username: str, password: str, request): # noqa: PLR6301 return password == HTTP_AUTH_USERS.get(username) -async def redirect(_): +async def redirect(_: web.Request) -> web.Response: return web.HTTPFound("/shop.tfl") -async def handler(_): +async def handler(_: web.Request) -> web.Response: data = json.dumps(generate_tfl_file()) return web.Response(text=data, content_type="application/json", charset="utf-8") @@ -40,7 +40,7 @@ async def prepare_server(host: str, port: int) -> tuple[web.TCPSite, web.AppRunn web.get("/", redirect), web.get("/shop.tfl", handler), web.static("/", NPS_DIR, follow_symlinks=True), - ] + ], ) if HTTP_AUTH_USERS: app.middlewares.append(BasicAuth()) diff --git a/poetry.lock b/poetry.lock index 9031194..9540680 100644 --- a/poetry.lock +++ b/poetry.lock @@ -787,5 +787,5 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" -python-versions = "^3.11" -content-hash = "a1ea9ee593828f292b915d19561dde6de80852a0fa07eda7c889971e6a511189" +python-versions = "^3.12" +content-hash = "fc9107e0f2e9405c69d4444324bb118450e6c200a27bf2e91e4958b6c64ec20f" diff --git a/pyproject.toml b/pyproject.toml index bb751ac..d94c498 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,19 @@ [tool.poetry] -name = "tinfoil" -version = "0.2.0" +name = "owntinfoil" +version = "0.2.1" description = "" -authors = ["jag-k"] +authors = ["jag-k "] +maintainers = ["jag-k "] license = "MIT" readme = "README.md" +homepage = "https://github.com/jag-k/owntinfoil" +repository = "https://github.com/jag-k/owntinfoil" +documentation = "https://github.com/jag-k/owntinfoil/wiki" +keywords = ["tinfoil", "owntinfoil", "nintendo-switch"] +packages = [] [tool.poetry.dependencies] -python = "^3.11" +python = "^3.12" python-dotenv = "^1.0.0" aiohttp = "^3.9.1" aiofile = "^3.8.5" @@ -22,36 +28,47 @@ ruff-lsp = "^0.0.45" requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" -[tool.black] -line-length = 120 -include = '\.pyi?$' -exclude = ''' -/( - \.git - | \.hg - | \.mypy_cache - | \.pytest_cache - | \.tox - | \.venv -)/ -''' - - [tool.ruff] line-length = 120 preview = true target-version = "py311" -exclude = ["*locales*"] + +[tool.ruff.lint] select = [ - "E", # pyflakes - "F", # pycodestyle errors + "F", # pyflakes + "E", # pycodestyle errors "W", # pycodestyle warnings - "UP", # pyupgrade "I", # isort + "N", # pep8-naming + "UP", # pyupgrade + "YTT", # flake8-2020 + # "ANN", # flake8-annotations + "ASYNC", # flake8-async + "TRIO", # flake8-trio + "S", # flake8-bandit + "B", # flake8-bugbear + "A", # flake8-builtins + "COM", # flake8-commas "C4", # flake8-comprehensions - # pytest - "PT018", # Assertion should be broken down into multiple parts - "PT022", # No teardown in fixture {name}, use return instead of yield + "ICN", # flake8-import-conventions + "G", # flake8-logging-format + "INP", # flake8-no-pep420 + "PIE", # flake8-pie + "PYI", # flake8-pyi + "RSE", # flake8-raise + "RET", # flake8-return + "SIM", # flake8-simplify + "TCH", # flake8-type-checking + "INT", # flake8-gettext + "PTH", # flake8-use-pathlib + "TD", # flake8-todos + "ERA", # eradicate + "PGH", # pygrep-hooks + "PL", # PyLint + "FLY", # flynt + "PERF", # Perflint + + "RUF", # ruff ] ignore = ["E501"]