diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml new file mode 100644 index 00000000..52b83cc8 --- /dev/null +++ b/.github/workflows/publish-docker-image.yml @@ -0,0 +1,43 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# GitHub recommends pinning actions to a commit SHA. +# To get a newer version, you will need to update the SHA. +# You can also reference a tag or branch, but the action may change without warning. + +name: Publish Docker image + +on: + release: + types: [published] + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: my-docker-hub-namespace/my-docker-hub-repository + + - name: Build and push Docker image + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4910b063..02e43d0c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,17 +40,16 @@ jobs: run: | python3 -m pip install -U pip setuptools python3 -m pip install poetry==1.7.1 - poetry config virtualenvs.create false poetry install --with=dev - name: Version info run: | - python main.py --version + poetry run python main.py --version - name: Test with pytest run: | - pytest + poetry run pytest - name: Check with MyPy run: | - mypy . + poetry run mypy . diff --git a/CHANGELOG.md b/CHANGELOG.md index 572ba02f..8a1e7317 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,10 @@ ## [Unreleased] -- Integrated `pdftotext`, `pdfminer` and `docx2txt` interfaces into `filecontent` filter. +- Integrated `.docx`, `.pdf` and various raw text parsers into `filecontent` filter. - Removed `textract` and ~50 MB of dependencies as they are no longer needed. - Python 3.12 support +- Add support for piping in a config file from STDIN (`organize run --stdin < file.yml`) ## v3.1.2 (2024-02-16) diff --git a/Dockerfile b/Dockerfile index 40941dec..ce7d9ab3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ RUN apt update && \ rm -rf /var/lib/apt/lists/* ENV ORGANIZE_CONFIG=/config/config.yml \ ORGANIZE_EXIFTOOL_PATH=exiftool -RUN mkdir /config && touch ./README.md +RUN mkdir /config && mkdir /data COPY --from=pydeps ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY ./organize ./organize diff --git a/compose.yml b/compose.yml deleted file mode 100644 index 01066468..00000000 --- a/compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3.8" -name: "organize" - -services: - organize: - build: . diff --git a/docker-conf.yml b/docker-conf.yml new file mode 100644 index 00000000..1b83ad42 --- /dev/null +++ b/docker-conf.yml @@ -0,0 +1,4 @@ +rules: + - locations: /data + actions: + - echo: "Found file: {path}" diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 00000000..7a1d350c --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,52 @@ +# Using the organize docker image + +The organize docker image comes preinstalled with `exiftool` and `pdftotext` as well as +all the python dependencies set up and ready to go. + +!!! danger + + As organize is mainly used for moving files around you have to be careful about your + volume mounts and paths. **If you move a file to a folder which is not persisted + it is gone as soon as the container is stopped!** + +## Building the image + +`cd` into the organize folder (containing the `Dockerfile`) and build the image: + +```sh +docker build -t organize . +``` + +The image is now tagged as `organize`. Now you can test the image by running + +```sh +docker run organize +``` + +This will show the organize usage help text. + +## Running + +Let's create a basic config file `docker-conf.yml`: + +```yml +rules: + - locations: /data + actions: + - echo: "Found file: {path}" +``` + +We can now run mount the config file to the container path `/config/config.yml`. The current directory is mounted to `/data` so we have some files present. +We can now start the container: + +```sh +docker run -v ./docker-conf.yml:/config/config.yml -v .:/data organize run +``` + +### Passing the config file from stdin + +Instead of mounting the config file into the container you can also pass it from stdin: + +```sh +docker run -i organize check --stdin < ./docker-conf.yml +``` diff --git a/mkdocs.yml b/mkdocs.yml index 34c35e58..511b16b3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,6 +9,7 @@ nav: - Locations: locations.md - Filters: filters.md - Actions: actions.md + - Docker: docker.md - Changelog: changelog.md - Migrating from older versions: migrating.md plugins: diff --git a/organize/cli.py b/organize/cli.py index 3ced34bd..a53d3118 100644 --- a/organize/cli.py +++ b/organize/cli.py @@ -2,12 +2,12 @@ organize - The file management automation tool. Usage: - organize run [options] [] - organize sim [options] [] + organize run [options] [ | --stdin] + organize sim [options] [ | --stdin] organize new [] organize edit [] - organize check [] - organize debug [] + organize check [ | --stdin] + organize debug [ | --stdin] organize show [--path|--reveal] [] organize list organize docs @@ -28,7 +28,9 @@ docs Open the documentation. Options: - A config name or path to a config file + A config name or path to a config file. + Some commands also support piping in a config file + via the `--stdin` flag. -W --working-dir The working directory -F --format (default|errorsonly|JSONL) The output format [Default: default] @@ -43,9 +45,17 @@ from typing import Annotated, Literal, Optional, Set from docopt import docopt -from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + field_validator, + model_validator, +) from pydantic.functional_validators import BeforeValidator from rich.console import Console +from rich.pretty import pprint from rich.syntax import Syntax from rich.table import Table from yaml.scanner import ScannerError @@ -79,6 +89,33 @@ console = Console() +class ConfigWithPath(BaseModel): + """ + Allows reading the config from a path, finding it by name or supplying it directly + via stdin. + """ + + config: str + config_path: Optional[Path] + + @classmethod + def from_stdin(cls) -> "ConfigWithPath": + return cls(config=sys.stdin.read(), config_path=None) + + @classmethod + def by_name_or_path(cls, name_or_path: Optional[str]) -> "ConfigWithPath": + config_path = find_config(name_or_path=name_or_path) + return cls( + config=config_path.read_text(), + config_path=config_path, + ) + + def path(self): + if self.config_path is not None: + return str(self.config_path) + return "[config given by string / stdin]" + + def _open_uri(uri: str) -> None: import webbrowser @@ -96,15 +133,17 @@ def _output_for_format(format: OutputFormat) -> Output: def execute( - config: Optional[str], + config: ConfigWithPath, working_dir: Optional[Path], format: OutputFormat, tags: Tags, skip_tags: Tags, simulate: bool, ) -> None: - config_path = find_config(name_or_path=config) - Config.from_path(config_path).execute( + Config.from_string( + config=config.config, + config_path=config.config_path, + ).execute( simulate=simulate, output=_output_for_format(format), tags=tags, @@ -138,21 +177,14 @@ def edit(config: Optional[str]) -> None: _open_uri(config_path.as_uri()) -def check(config: Optional[str]) -> None: - config_path = find_config(config) - Config.from_path(config_path=config_path) - console.print(f'No problems found in "{escape(config_path)}".') +def check(config: ConfigWithPath) -> None: + Config.from_string(config=config.config, config_path=config.config_path) + console.print(f'No problems found in "{escape(config.path())}".') -def debug(config: Optional[str]) -> None: - from rich.pretty import pprint - - config_path = find_config(config) - pprint( - Config.from_path(config_path=config_path), - expand_all=True, - indent_guides=False, - ) +def debug(config: ConfigWithPath) -> None: + conf = Config.from_string(config=config.config, config_path=config.config_path) + pprint(conf, expand_all=True, indent_guides=False) def show(config: Optional[str], path: bool, reveal: bool) -> None: @@ -201,6 +233,7 @@ class CliArgs(BaseModel): format: OutputFormat = Field("default", alias="--format") tags: Optional[str] = Field(..., alias="--tags") skip_tags: Optional[str] = Field(..., alias="--skip-tags") + stdin: bool = Field(..., alias="--stdin") # show options path: bool = Field(False, alias="--path") @@ -217,6 +250,12 @@ def split_tags(cls, val) -> Set[str]: return set() return set(val.split(",")) + @model_validator(mode="after") + def either_stdin_or_config(self): + if self.stdin and self.config is not None: + raise ValueError("Either set a config file or --stdin.") + return self + def cli() -> None: arguments = docopt( @@ -226,9 +265,13 @@ def cli() -> None: ) try: args = CliArgs.model_validate(arguments) + if args.stdin: + config_with_path = ConfigWithPath.from_stdin() + else: + config_with_path = ConfigWithPath.by_name_or_path(args.config) _execute = partial( execute, - config=args.config, + config=config_with_path, working_dir=args.working_dir, format=args.format, tags=args.tags, @@ -243,9 +286,9 @@ def cli() -> None: elif args.edit: edit(config=args.config) elif args.check: - check(config=args.config) + check(config=config_with_path) elif args.debug: - debug(config=args.config) + debug(config=config_with_path) elif args.show: show(config=args.config, path=args.path, reveal=args.reveal) elif args.list: diff --git a/organize/config.py b/organize/config.py index c53b57c5..cc36c36d 100644 --- a/organize/config.py +++ b/organize/config.py @@ -64,6 +64,8 @@ def from_string(cls, config: str, config_path: Optional[Path] = None) -> Config: dedented = textwrap.dedent(normalized) as_dict = yaml.load(dedented, Loader=yaml.SafeLoader) try: + if not as_dict: + raise ValueError("Config is empty") inst = cls(**as_dict) inst._config_path = config_path return inst