Merge branch 'feat/stdin'

.github/workflows/publish-docker-image.yml

@@ -0,0 +1,43 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: Publish Docker image
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: my-docker-hub-namespace/my-docker-hub-repository
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/tests.yml

@@ -40,17 +40,16 @@ jobs:
         run: |
           python3 -m pip install -U pip setuptools
           python3 -m pip install poetry==1.7.1
-          poetry config virtualenvs.create false
           poetry install --with=dev
 
       - name: Version info
         run: |
-          python main.py --version
+          poetry run python main.py --version
 
       - name: Test with pytest
         run: |
-          pytest
+          poetry run pytest
 
       - name: Check with MyPy
         run: |
-          mypy .
+          poetry run mypy .

CHANGELOG.md

@@ -2,9 +2,10 @@
 
 ## [Unreleased]
 
-- Integrated `pdftotext`, `pdfminer` and `docx2txt` interfaces into `filecontent` filter.
+- Integrated `.docx`, `.pdf` and various raw text parsers into `filecontent` filter.
 - Removed `textract` and ~50 MB of dependencies as they are no longer needed.
 - Python 3.12 support
+- Add support for piping in a config file from STDIN (`organize run --stdin < file.yml`)
 
 ## v3.1.2 (2024-02-16)
 

Dockerfile

@@ -22,7 +22,7 @@ RUN apt update && \
     rm -rf /var/lib/apt/lists/*
 ENV ORGANIZE_CONFIG=/config/config.yml \
     ORGANIZE_EXIFTOOL_PATH=exiftool
-RUN mkdir /config && touch ./README.md
+RUN mkdir /config && mkdir /data
 COPY --from=pydeps ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 COPY ./organize ./organize
 

docker-conf.yml

@@ -0,0 +1,4 @@
+rules:
+  - locations: /data
+    actions:
+      - echo: "Found file: {path}"

docs/docker.md

@@ -0,0 +1,52 @@
+# Using the organize docker image
+
+The organize docker image comes preinstalled with `exiftool` and `pdftotext` as well as
+all the python dependencies set up and ready to go.
+
+!!! danger
+
+    As organize is mainly used for moving files around you have to be careful about your
+    volume mounts and paths. **If you move a file to a folder which is not persisted
+    it is gone as soon as the container is stopped!**
+
+## Building the image
+
+`cd` into the organize folder (containing the `Dockerfile`) and build the image:
+
+```sh
+docker build -t organize .
+```
+
+The image is now tagged as `organize`. Now you can test the image by running
+
+```sh
+docker run organize
+```
+
+This will show the organize usage help text.
+
+## Running
+
+Let's create a basic config file `docker-conf.yml`:
+
+```yml
+rules:
+  - locations: /data
+    actions:
+      - echo: "Found file: {path}"
+```
+
+We can now run mount the config file to the container path `/config/config.yml`. The current directory is mounted to `/data` so we have some files present.
+We can now start the container:
+
+```sh
+docker run -v ./docker-conf.yml:/config/config.yml -v .:/data organize run
+```
+
+### Passing the config file from stdin
+
+Instead of mounting the config file into the container you can also pass it from stdin:
+
+```sh
+docker run -i organize check --stdin < ./docker-conf.yml
+```

mkdocs.yml

@@ -9,6 +9,7 @@ nav:
   - Locations: locations.md
   - Filters: filters.md
   - Actions: actions.md
+  - Docker: docker.md
   - Changelog: changelog.md
   - Migrating from older versions: migrating.md
 plugins:

organize/cli.py

@@ -2,12 +2,12 @@
 organize - The file management automation tool.
 
 Usage:
-  organize run    [options] [<config>]
-  organize sim    [options] [<config>]
+  organize run    [options] [<config> | --stdin]
+  organize sim    [options] [<config> | --stdin]
   organize new    [<config>]
   organize edit   [<config>]
-  organize check  [<config>]
-  organize debug  [<config>]
+  organize check  [<config> | --stdin]
+  organize debug  [<config> | --stdin]
   organize show   [--path|--reveal] [<config>]
   organize list
   organize docs
@@ -28,7 +28,9 @@
   docs       Open the documentation.
 
 Options:
-  <config>                        A config name or path to a config file
+  <config>                        A config name or path to a config file.
+                                  Some commands also support piping in a config file
+                                  via the `--stdin` flag.
   -W --working-dir <dir>          The working directory
   -F --format (default|errorsonly|JSONL)
                                   The output format [Default: default]
@@ -43,9 +45,17 @@
 from typing import Annotated, Literal, Optional, Set
 
 from docopt import docopt
-from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    ValidationError,
+    field_validator,
+    model_validator,
+)
 from pydantic.functional_validators import BeforeValidator
 from rich.console import Console
+from rich.pretty import pprint
 from rich.syntax import Syntax
 from rich.table import Table
 from yaml.scanner import ScannerError
@@ -79,6 +89,33 @@
 console = Console()
 
 
+class ConfigWithPath(BaseModel):
+    """
+    Allows reading the config from a path, finding it by name or supplying it directly
+    via stdin.
+    """
+
+    config: str
+    config_path: Optional[Path]
+
+    @classmethod
+    def from_stdin(cls) -> "ConfigWithPath":
+        return cls(config=sys.stdin.read(), config_path=None)
+
+    @classmethod
+    def by_name_or_path(cls, name_or_path: Optional[str]) -> "ConfigWithPath":
+        config_path = find_config(name_or_path=name_or_path)
+        return cls(
+            config=config_path.read_text(),
+            config_path=config_path,
+        )
+
+    def path(self):
+        if self.config_path is not None:
+            return str(self.config_path)
+        return "[config given by string / stdin]"
+
+
 def _open_uri(uri: str) -> None:
     import webbrowser
 
@@ -96,15 +133,17 @@ def _output_for_format(format: OutputFormat) -> Output:
 
 
 def execute(
-    config: Optional[str],
+    config: ConfigWithPath,
     working_dir: Optional[Path],
     format: OutputFormat,
     tags: Tags,
     skip_tags: Tags,
     simulate: bool,
 ) -> None:
-    config_path = find_config(name_or_path=config)
-    Config.from_path(config_path).execute(
+    Config.from_string(
+        config=config.config,
+        config_path=config.config_path,
+    ).execute(
         simulate=simulate,
         output=_output_for_format(format),
         tags=tags,
@@ -138,21 +177,14 @@ def edit(config: Optional[str]) -> None:
         _open_uri(config_path.as_uri())
 
 
-def check(config: Optional[str]) -> None:
-    config_path = find_config(config)
-    Config.from_path(config_path=config_path)
-    console.print(f'No problems found in "{escape(config_path)}".')
+def check(config: ConfigWithPath) -> None:
+    Config.from_string(config=config.config, config_path=config.config_path)
+    console.print(f'No problems found in "{escape(config.path())}".')
 
 
-def debug(config: Optional[str]) -> None:
-    from rich.pretty import pprint
-
-    config_path = find_config(config)
-    pprint(
-        Config.from_path(config_path=config_path),
-        expand_all=True,
-        indent_guides=False,
-    )
+def debug(config: ConfigWithPath) -> None:
+    conf = Config.from_string(config=config.config, config_path=config.config_path)
+    pprint(conf, expand_all=True, indent_guides=False)
 
 
 def show(config: Optional[str], path: bool, reveal: bool) -> None:
@@ -201,6 +233,7 @@ class CliArgs(BaseModel):
     format: OutputFormat = Field("default", alias="--format")
     tags: Optional[str] = Field(..., alias="--tags")
     skip_tags: Optional[str] = Field(..., alias="--skip-tags")
+    stdin: bool = Field(..., alias="--stdin")
 
     # show options
     path: bool = Field(False, alias="--path")
@@ -217,6 +250,12 @@ def split_tags(cls, val) -> Set[str]:
             return set()
         return set(val.split(","))
 
+    @model_validator(mode="after")
+    def either_stdin_or_config(self):
+        if self.stdin and self.config is not None:
+            raise ValueError("Either set a config file or --stdin.")
+        return self
+
 
 def cli() -> None:
     arguments = docopt(
@@ -226,9 +265,13 @@ def cli() -> None:
     )
     try:
         args = CliArgs.model_validate(arguments)
+        if args.stdin:
+            config_with_path = ConfigWithPath.from_stdin()
+        else:
+            config_with_path = ConfigWithPath.by_name_or_path(args.config)
         _execute = partial(
             execute,
-            config=args.config,
+            config=config_with_path,
             working_dir=args.working_dir,
             format=args.format,
             tags=args.tags,
@@ -243,9 +286,9 @@ def cli() -> None:
         elif args.edit:
             edit(config=args.config)
         elif args.check:
-            check(config=args.config)
+            check(config=config_with_path)
         elif args.debug:
-            debug(config=args.config)
+            debug(config=config_with_path)
         elif args.show:
             show(config=args.config, path=args.path, reveal=args.reveal)
         elif args.list:

organize/config.py

@@ -64,6 +64,8 @@ def from_string(cls, config: str, config_path: Optional[Path] = None) -> Config:
         dedented = textwrap.dedent(normalized)
         as_dict = yaml.load(dedented, Loader=yaml.SafeLoader)
         try:
+            if not as_dict:
+                raise ValueError("Config is empty")
             inst = cls(**as_dict)
             inst._config_path = config_path
             return inst