From 6a25a7358067e64d7291150d82ede01446fdee2b Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 12:13:22 +0000 Subject: [PATCH 01/12] misc. --- .github/workflows/docker_smoketest.yml | 13 ++++-- docker_smoketest.sh | 21 --------- memium/subtasks/smoketest.py | 59 ++++++++++++++++++++++++++ memium/test_cli.py | 17 ++++++++ readme.md | 2 +- tasks.py | 4 ++ 6 files changed, 91 insertions(+), 25 deletions(-) delete mode 100755 docker_smoketest.sh create mode 100644 memium/subtasks/smoketest.py create mode 100644 memium/test_cli.py diff --git a/.github/workflows/docker_smoketest.yml b/.github/workflows/docker_smoketest.yml index 989f7a4..ac7494f 100644 --- a/.github/workflows/docker_smoketest.yml +++ b/.github/workflows/docker_smoketest.yml @@ -16,6 +16,10 @@ jobs: group: ${{ github.repository }}-${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || ''}} cancel-in-progress: true runs-on: ubuntu-latest + strategy: + matrix: + invoke_command: ["smoketest-docker", "smoketest-cli"] + steps: - name: Checkout (GitHub) uses: actions/checkout@v4 @@ -27,9 +31,12 @@ jobs: username: MartinBernstorff password: ${{ secrets.GITHUB_TOKEN }} + - name: Install invoke + run: | + python -m pip install --upgrade pip + pip install invoke + - name: Run integration test shell: bash run: | - TEST_PATH=./docker_smoketest.sh - chmod +x $TEST_PATH - $TEST_PATH \ No newline at end of file + inv ${{ matrix.invoke_command }} \ No newline at end of file diff --git a/docker_smoketest.sh b/docker_smoketest.sh deleted file mode 100755 index 508f9dd..0000000 --- a/docker_smoketest.sh +++ /dev/null @@ -1,21 +0,0 @@ -set -e -docker build . -t memium:latest -f Dockerfile -docker volume create ankidecks - -INPUT_DIR=$HOME/input/ - -mkdir -p $INPUT_DIR -echo "Creating $INPUT_DIR" -echo -e "Q. Question here\nA. Answer!" >> $INPUT_DIR/test.md - -docker run -i \ - -e HOST_INPUT_DIR=$INPUT_DIR \ - -v $INPUT_DIR:/input \ - --restart unless-stopped \ - memium \ - memium \ - --input-dir /input \ - --dry-run \ - --skip-sync - -echo "Smoketesting succesful!" \ No newline at end of file diff --git a/memium/subtasks/smoketest.py b/memium/subtasks/smoketest.py new file mode 100644 index 0000000..3fa4f88 --- /dev/null +++ b/memium/subtasks/smoketest.py @@ -0,0 +1,59 @@ +from collections.abc import Sequence +from pathlib import Path + +import invoke as inv + +from memium.subtasks.graphite import submit_pr # noqa: F401 # type: ignore + + +def get_code_blocks_from_md(md_path: Path) -> Sequence[str]: + md = md_path.read_text() + code_blocks = md.split("```")[1::2] + return code_blocks + + +def create_smoketest_dir() -> Path: + print("💨 Running smoketest") + input_dir = Path.home() / "smoketest_dir" + input_dir.mkdir(exist_ok=True) + + test_file = input_dir / "test.md" + test_file.write_text("Q. Question here\nA. Answer!") + return input_dir + + +@inv.task # type: ignore +def smoketest_docker(c: inv.Context): + input_dir = create_smoketest_dir() + + code_blocks = get_code_blocks_from_md(Path("README.md")) + docker_block = next(block for block in code_blocks if "docker run" in block) + + # Only keep content after docker line + docker_block = docker_block[docker_block.index("docker run") :] + replaced_input_dir = docker_block.replace("$INPUT_DIR", str(input_dir)) + + smoketest_docker_command = ( + replaced_input_dir + " --dry-run \\\n" + " --skip-sync" + ) + + print(smoketest_docker_command) + + c.run("docker build . -t memium:latest -f Dockerfile") + c.run("docker volume create ankidecks") + c.run(smoketest_docker_command) + print("💨🎉 Smoketest complete") + + +@inv.task # type: ignore +def smoketest_cli(c: inv.Context): + smoketest_dir = create_smoketest_dir() + code_blocks = get_code_blocks_from_md(Path("README.md")) + cli_block = next(block for block in code_blocks if "cli-block" in block) + sanitised_cli_block = cli_block.splitlines()[1].replace("> ", "") + + cli_smoketest_cmd = sanitised_cli_block.replace( + "[YOUR_INPUT_DIR]", str(smoketest_dir) + ) + print(cli_smoketest_cmd) + c.run(cli_smoketest_cmd) diff --git a/memium/test_cli.py b/memium/test_cli.py new file mode 100644 index 0000000..46e44fe --- /dev/null +++ b/memium/test_cli.py @@ -0,0 +1,17 @@ +from pathlib import Path + +from .cli import cli + + +def test_cli(tmp_path: Path): + test_file = tmp_path / "test.md" + test_file.write_text("Q. Question here\nA. Answer!") + cli( + input_dir=Path("/root/input"), + watch=None, + deck_name="Test CLI", + max_deletions_per_run=50, + push_all=False, + dry_run=False, + sentry_dsn=None, + ) diff --git a/readme.md b/readme.md index 5a48028..496790b 100644 --- a/readme.md +++ b/readme.md @@ -42,7 +42,7 @@ If you want to sync markdown notes to Anki, here's how to get started! 3. Import your notes! -```bash +```cli-block > memium --input-dir [YOUR_INPUT_DIR] ``` diff --git a/tasks.py b/tasks.py index 9a9ad5f..6902f8a 100644 --- a/tasks.py +++ b/tasks.py @@ -5,6 +5,10 @@ create_branch_from_issue, submit_pr, # noqa: F401 # type: ignore ) +from memium.subtasks.smoketest import ( + smoketest_cli, # noqa: F401 # type: ignore + smoketest_docker, # noqa: F401 # type: ignore +) PYTEST_CMD = "pytest --durations=5 --cov=memium memium --cov-report xml:.coverage.xml --cov-report lcov:.coverage.lcov" From d9bba4189697b05960070d9fbde8384e26c20e3e Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 11:39:28 +0000 Subject: [PATCH 02/12] tests: extract smoketest command from readme Fixes #463 From 3c208c0004d66a881051355ea7b725f9b373f905 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 12:15:02 +0000 Subject: [PATCH 03/12] misc. --- .github/workflows/docker_smoketest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker_smoketest.yml b/.github/workflows/docker_smoketest.yml index ac7494f..bda0ce5 100644 --- a/.github/workflows/docker_smoketest.yml +++ b/.github/workflows/docker_smoketest.yml @@ -39,4 +39,5 @@ jobs: - name: Run integration test shell: bash run: | + pip install . inv ${{ matrix.invoke_command }} \ No newline at end of file From 35828d56a6bfae18f84cd1ae7fe8034643d8ed3a Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 12:15:48 +0000 Subject: [PATCH 04/12] fix: readme casing --- .devcontainer/devcontainer.json | 2 +- memium/subtasks/smoketest.py | 4 ++-- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 41ff972..048e43c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the -// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile +// readme at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile { "name": "Existing Dockerfile", "build": { diff --git a/memium/subtasks/smoketest.py b/memium/subtasks/smoketest.py index 3fa4f88..f9e4a4f 100644 --- a/memium/subtasks/smoketest.py +++ b/memium/subtasks/smoketest.py @@ -26,7 +26,7 @@ def create_smoketest_dir() -> Path: def smoketest_docker(c: inv.Context): input_dir = create_smoketest_dir() - code_blocks = get_code_blocks_from_md(Path("README.md")) + code_blocks = get_code_blocks_from_md(Path("readme.md")) docker_block = next(block for block in code_blocks if "docker run" in block) # Only keep content after docker line @@ -48,7 +48,7 @@ def smoketest_docker(c: inv.Context): @inv.task # type: ignore def smoketest_cli(c: inv.Context): smoketest_dir = create_smoketest_dir() - code_blocks = get_code_blocks_from_md(Path("README.md")) + code_blocks = get_code_blocks_from_md(Path("readme.md")) cli_block = next(block for block in code_blocks if "cli-block" in block) sanitised_cli_block = cli_block.splitlines()[1].replace("> ", "") diff --git a/pyproject.toml b/pyproject.toml index b2f5ce4..568b667 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ file = "LICENSE" [project.readme] -file = "README.md" +file = "readme.md" content-type = "text/markdown" [project.scripts] From a87d0513d020efee80b2ff3765dc216fc1bdb624 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 12:16:12 +0000 Subject: [PATCH 05/12] misc. --- .github/workflows/{docker_smoketest.yml => smoketests.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{docker_smoketest.yml => smoketests.yml} (97%) diff --git a/.github/workflows/docker_smoketest.yml b/.github/workflows/smoketests.yml similarity index 97% rename from .github/workflows/docker_smoketest.yml rename to .github/workflows/smoketests.yml index bda0ce5..2c828b6 100644 --- a/.github/workflows/docker_smoketest.yml +++ b/.github/workflows/smoketests.yml @@ -2,7 +2,7 @@ # then it will in python 3.9 (ubuntu-latest) create a badge with the coverage # and add it to the PR. This badge will be updated if the PR is updated. -name: docker-smoketest +name: smoketests on: push: branches: [main] @@ -10,7 +10,7 @@ on: branches: [main] jobs: - smoketest: + smoketests: permissions: write-all concurrency: group: ${{ github.repository }}-${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || ''}} From d9dfd1abf032688df7b3bf0e9f201598e2fe1bae Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 12:16:46 +0000 Subject: [PATCH 06/12] misc. --- memium/subtasks/smoketest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/memium/subtasks/smoketest.py b/memium/subtasks/smoketest.py index f9e4a4f..ea89192 100644 --- a/memium/subtasks/smoketest.py +++ b/memium/subtasks/smoketest.py @@ -56,4 +56,5 @@ def smoketest_cli(c: inv.Context): "[YOUR_INPUT_DIR]", str(smoketest_dir) ) print(cli_smoketest_cmd) - c.run(cli_smoketest_cmd) + c.run(cli_smoketest_cmd + " --dry-run \\\n" + " --skip-sync") + print("💨🎉 Smoketest complete") From 77169692f7fcc1ed7a34571f51ba1732bc4bdba2 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 12:17:20 +0000 Subject: [PATCH 07/12] concurrency --- .github/workflows/smoketests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoketests.yml b/.github/workflows/smoketests.yml index 2c828b6..40dfea9 100644 --- a/.github/workflows/smoketests.yml +++ b/.github/workflows/smoketests.yml @@ -13,7 +13,7 @@ jobs: smoketests: permissions: write-all concurrency: - group: ${{ github.repository }}-${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || ''}} + group: ${{ github.repository }}-${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || ''}}-${{ matrix.invoke_command }} cancel-in-progress: true runs-on: ubuntu-latest strategy: From 02d97d83ec1b10c4b0af4fc1a8efb4839636680b Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 12:18:00 +0000 Subject: [PATCH 08/12] misc: setup python --- .github/workflows/smoketests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/smoketests.yml b/.github/workflows/smoketests.yml index 40dfea9..a011fdd 100644 --- a/.github/workflows/smoketests.yml +++ b/.github/workflows/smoketests.yml @@ -30,6 +30,11 @@ jobs: registry: ghcr.io username: MartinBernstorff password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.11 - name: Install invoke run: | From 6ed9ca093cf6a2641ae694039e7b5e25ccfaf871 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 12:21:36 +0000 Subject: [PATCH 09/12] tests: remove unused test --- memium/test_cli.py | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 memium/test_cli.py diff --git a/memium/test_cli.py b/memium/test_cli.py deleted file mode 100644 index 46e44fe..0000000 --- a/memium/test_cli.py +++ /dev/null @@ -1,17 +0,0 @@ -from pathlib import Path - -from .cli import cli - - -def test_cli(tmp_path: Path): - test_file = tmp_path / "test.md" - test_file.write_text("Q. Question here\nA. Answer!") - cli( - input_dir=Path("/root/input"), - watch=None, - deck_name="Test CLI", - max_deletions_per_run=50, - push_all=False, - dry_run=False, - sentry_dsn=None, - ) From 1cef659d0585906eeed7f457f6a3d0c21ef8b1c0 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 11:39:28 +0000 Subject: [PATCH 10/12] tests: extract smoketest command from readme Fixes #463 From e0f3ba2846db2433b87bac5ef01e55b159f62a26 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 12:23:50 +0000 Subject: [PATCH 11/12] misc. --- .vscode/tasks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 29f92e2..069ce30 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -86,7 +86,7 @@ { "label": "Update from ancestor", "type": "shell", - "command": "gt restack --upstack", + "command": "gt restack", "presentation": { "reveal": "always", "clear": true, From edf75b6f531b10c57f847da40eb5204bccae39e8 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Wed, 27 Dec 2023 12:24:56 +0000 Subject: [PATCH 12/12] misc. --- .vscode/tasks.json | 2 +- memium/subtasks/graphite.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 069ce30..4e2974c 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -86,7 +86,7 @@ { "label": "Update from ancestor", "type": "shell", - "command": "gt restack", + "command": "gt sync --delete --force", "presentation": { "reveal": "always", "clear": true, diff --git a/memium/subtasks/graphite.py b/memium/subtasks/graphite.py index 8985acb..f0e46d7 100644 --- a/memium/subtasks/graphite.py +++ b/memium/subtasks/graphite.py @@ -5,7 +5,6 @@ @inv.task(aliases=("submit",)) # type: ignore def submit_pr(c: inv.Context): - c.run("gt sync --delete --force --no-restack") c.run("gt submit -m --no-edit --publish") c.run("gt log short")