Merge branch 'master' into snyk-fix-5ef3afe3fbfad34ca892e17f8d68fd7a

.drone/github-mirror.sh

@@ -6,9 +6,18 @@
 ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
 
 # Clone the repository
+branch=$(git rev-parse --abbrev-ref HEAD)
+if [ -z "${branch}" ]; then
+  echo "No branch checked out"
+  exit 1
+fi
+
 git remote add github [email protected]:/blacklight/platypush.git
-git pull --rebase github "$(git branch | head -1 | awk '{print $2}')" || echo "No such branch on Github"
+
+if (( "$branch" == "master" )); then
+  git pull --rebase github "${branch}" || echo "No such branch on Github"
+fi
 
 # Push the changes to the GitHub mirror
-git push --all -v github
+git push -f --all -v github
 git push --tags -v github

.ignore

@@ -0,0 +1 @@
+dist/

CHANGELOG.md

@@ -1,5 +1,65 @@
 # Changelog
 
+## [1.3.1]
+
+- [[#344](https://git.platypush.tech/platypush/platypush/issues/344)]: removed
+  `marshmallow_dataclass` dependency. That package isn't included in the
+  package managers of any supported distros and requires to be installed via
+  pip. Making the Platypush' system packages depend on a pip-only package is
+  not a good idea. Plus, the library seems to be still in heavy development and
+  it has already broken compatibility with at least the `system` package.
+
+## [1.3.0]
+
+- [[#333](https://git.platypush.tech/platypush/platypush/issues/333)]: new file
+  browser UI/component. It includes custom MIME type support, a file editor
+  with syntax highlight, file download and file upload.
+
+- [[#341](https://git.platypush.tech/platypush/platypush/issues/341)]:
+  procedures are now native entities that can be managed from the entities panel.
+  A new versatile procedure editor has also been added, with support for nested
+  blocks, conditions, loops, variables, context autocomplete, and more.
+
+- [`procedure`]: Added the following features to YAML/structured procedures:
+
+  - `set`: to set variables whose scope is limited to the procedure / code
+    block where they are created. `variable.set` is useful to permanently
+    store variables on the db, `variable.mset` is useful to set temporary
+    global variables in memory through Redis, but sometimes you may just want
+    to assign a value to a variable that only needs to live within a procedure,
+    event hook or cron.
+
+    ```yaml
+    - set:
+        foo: bar
+        temperature: ${output.get('temperature')}
+    ```
+
+  - `return` can now return values too when invoked within a procedure:
+
+      ```yaml
+      - return: something
+      # Or
+      - return: "Result: ${output.get('response')}"
+      ```
+
+- The default logging format is now much more compact. The full body of events
+  and requests is no longer included by default in `info` mode - instead, a
+  summary with the message type, ID and response time is logged. The full
+  payloads can still be logged by enabling `debug` logs through e.g. `-v`.
+
+## [1.2.3]
+
+- [[#422](https://git.platypush.tech/platypush/platypush/issues/422)]: adapted
+  media plugins to support streaming from the yt-dlp process. This allows
+  videos to have merged audio+video even if they had separate tracks upstream.
+
+- [`media.*`] Many improvements on the media UI.
+
+- [`zigbee.mqtt`] Removed synchronous logic from `zigbee.mqtt.device_set`. It
+  was prone to timeouts as well as pointless - the updated device state will
+  anyway be received as an event.
+
 ## [1.2.2]
 
 - Fixed regression on older version of Python that don't fully support

docker-compose.yml

@@ -1,14 +1,29 @@
 services:
   platypush:
+    # Replace the build section with the next line if instead of building the
+    # image from a local checkout you want to pull the latest base
+    # (Alpine-based) image from the remote registry
+    # image: "registry.platypush.tech/platypush:latest"
+
+    build:
+      context: .
+      # Alpine base image
+      dockerfile: ./platypush/install/docker/alpine.Dockerfile
+      # Debian base image
+      # dockerfile: ./platypush/install/docker/debian.Dockerfile
+      # Ubuntu base image
+      # dockerfile: ./platypush/install/docker/ubuntu.Dockerfile
+      # Fedora base image
+      # dockerfile: ./platypush/install/docker/fedora.Dockerfile
+
     restart: "always"
     command:
       - platypush
-      # Comment --start-redis if you want to run an external Redis service
-      # In such case you'll also have to ensure that the appropriate Redis
-      # variables are set in the .env file, or the Redis configuration is
-      # passed in the config.yaml, or use the --redis-host and --redis-port
-      # command-line options
-      - --start-redis
+      - --redis-host
+      - redis
+      # Or, if you want to run Redis from the same container as Platypush,
+      # replace --redis-host redis with the line below
+      # - --start-redis
 
     # Custom list of host devices that should be accessible to the container -
     # e.g. an Arduino, an ESP-compatible microcontroller, a joystick etc.
@@ -19,18 +34,7 @@ services:
     # (e.g. Bluetooth BLE or GPIO/SPI/I2C) if access to individual devices is
     # not enough or isn't practical
     # privileged: true
-
-    build:
-      context: .
-      # Alpine base image
-      dockerfile: ./platypush/install/docker/alpine.Dockerfile
-      # Debian base image
-      # dockerfile: ./platypush/install/docker/debian.Dockerfile
-      # Ubuntu base image
-      # dockerfile: ./platypush/install/docker/ubuntu.Dockerfile
-      # Fedora base image
-      # dockerfile: ./platypush/install/docker/fedora.Dockerfile
-
+
     # Copy .env.example to .env and modify as needed
     # env_file:
     #   - .env
@@ -40,7 +44,13 @@ services:
       # expose it
       - "8008:8008"
 
-    volumes:
-      - /path/to/your/config.yaml:/etc/platypush
-      - /path/to/a/workdir:/var/lib/platypush
+    # volumes:
+      # Replace with a path that contains/will contain your config.yaml file
+      # - /path/to/your/config:/etc/platypush
+      # Replace with a path that contains/will contain your working directory
+      # - /path/to/a/workdir:/var/lib/platypush
+      # Optionally, use an external volume for the cache
       # - /path/to/a/cachedir:/var/cache/platypush
+
+  redis:
+    image: redis

docs/source/platypush/plugins/procedures.rst

@@ -0,0 +1,5 @@
+``procedures``
+==============
+
+.. automodule:: platypush.plugins.procedures
+    :members:

docs/source/plugins.rst

@@ -77,7 +77,6 @@ Plugins
     platypush/plugins/media.kodi.rst
     platypush/plugins/media.mplayer.rst
     platypush/plugins/media.mpv.rst
-    platypush/plugins/media.omxplayer.rst
     platypush/plugins/media.plex.rst
     platypush/plugins/media.subtitles.rst
     platypush/plugins/media.vlc.rst
@@ -100,6 +99,7 @@ Plugins
     platypush/plugins/otp.rst
     platypush/plugins/pihole.rst
     platypush/plugins/ping.rst
+    platypush/plugins/procedures.rst
     platypush/plugins/pushbullet.rst
     platypush/plugins/pwm.pca9685.rst
     platypush/plugins/qrcode.rst

platypush/__init__.py

@@ -21,7 +21,7 @@
 # see https://git.platypush.tech/platypush/platypush/issues/399
 when = hook
 
-__version__ = '1.2.2'
+__version__ = '1.3.1'
 __author__ = 'Fabio Manganiello <[email protected]>'
 __all__ = [
     'Application',

platypush/app/_app.py

@@ -365,7 +365,13 @@ def _f(msg):
             elif isinstance(msg, Response):
                 msg.log()
             elif isinstance(msg, Event):
-                msg.log()
+                log.info(
+                    'Received event: %s.%s[id=%s]',
+                    msg.__class__.__module__,
+                    msg.__class__.__name__,
+                    msg.id,
+                )
+                msg.log(level=logging.DEBUG)
                 self.event_processor.process_event(msg)
 
         return _f

platypush/backend/http/app/streaming/plugins/file.py

@@ -1,7 +1,8 @@
 import os
+import pathlib
 from contextlib import contextmanager
 from datetime import datetime as dt
-from typing import Optional, Tuple
+from typing import IO, Optional, Tuple
 
 from tornado.web import stream_request_body
 
@@ -17,6 +18,8 @@ class FileRoute(StreamingRoute):
     """
 
     BUFSIZE = 1024
+    _bytes_written = 0
+    _out_f: Optional[IO[bytes]] = None
 
     @classmethod
     def path(cls) -> str:
@@ -39,6 +42,10 @@ def file_path(self) -> str:
     def file_size(self) -> int:
         return os.path.getsize(self.file_path)
 
+    @property
+    def _content_length(self) -> int:
+        return int(self.request.headers.get('Content-Length', 0))
+
     @property
     def range(self) -> Tuple[Optional[int], Optional[int]]:
         range_hdr = self.request.headers.get('Range')
@@ -105,6 +112,77 @@ def _serve(self):
 
         self.finish()
 
+    def on_finish(self) -> None:
+        if self._out_f:
+            try:
+                if not (self._out_f and self._out_f.closed):
+                    self._out_f.close()
+            except Exception as e:
+                self.logger.warning('Error while closing the output file: %s', e)
+
+            self._out_f = None
+
+        return super().on_finish()
+
+    def _validate_upload(self, force: bool = False) -> bool:
+        if not self.file_path:
+            self.write_error(400, 'Missing path argument')
+            return False
+
+        if not self._out_f:
+            if not force and os.path.exists(self.file_path):
+                self.write_error(409, f'{self.file_path} already exists')
+                return False
+
+            self._bytes_written = 0
+            dir_path = os.path.dirname(self.file_path)
+
+            try:
+                pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True)
+                self._out_f = open(  # pylint: disable=consider-using-with
+                    self.file_path, 'wb'
+                )
+            except PermissionError:
+                self.write_error(403, 'Permission denied')
+                return False
+
+        return True
+
+    def finish(self, *args, **kwargs):  # type: ignore
+        try:
+            return super().finish(*args, **kwargs)
+        except Exception as e:
+            self.logger.warning('Error while finishing the request: %s', e)
+
+    def data_received(self, chunk: bytes):
+        # Ignore unless we're in POST/PUT mode
+        if self.request.method not in ('POST', 'PUT'):
+            return
+
+        force = self.request.method == 'PUT'
+        if not self._validate_upload(force=force):
+            self.finish()
+            return
+
+        if not chunk:
+            self.logger.debug('Received EOF from client')
+            self.finish()
+            return
+
+        assert self._out_f
+        self._out_f.write(chunk)
+        self._out_f.flush()
+        self._bytes_written += len(chunk)
+        self.logger.debug(
+            'Written chunk of size %d to %s, progress: %d/%d',
+            len(chunk),
+            self.file_path,
+            self._bytes_written,
+            self._content_length,
+        )
+
+        self.flush()
+
     def get(self) -> None:
         with self._serve() as f:
             if f:
@@ -119,3 +197,9 @@ def get(self) -> None:
     def head(self) -> None:
         with self._serve():
             pass
+
+    def post(self) -> None:
+        self.logger.info('Receiving file POST upload request for %r', self.file_path)
+
+    def put(self) -> None:
+        self.logger.info('Receiving file PUT upload request for %r', self.file_path)

platypush/backend/http/app/streaming/plugins/media/_register.py

@@ -3,7 +3,9 @@
 from platypush.backend.http.app.utils import logger, send_request
 from platypush.backend.http.media.handlers import MediaHandler
 
-from ._registry import load_media_map, save_media_map
+from ._registry import clear_media_map, load_media_map, save_media_map
+
+_init = False
 
 
 def get_media_url(media_id: str) -> str:
@@ -17,6 +19,12 @@ def register_media(source: str, subtitles: Optional[str] = None) -> MediaHandler
     """
     Registers a media file and returns its associated media handler.
     """
+    global _init
+
+    if not _init:
+        clear_media_map()
+        _init = True
+
     media_id = MediaHandler.get_media_id(source)
     media_url = get_media_url(media_id)
     media_map = load_media_map()

platypush/backend/http/app/streaming/plugins/media/_registry.py

@@ -25,10 +25,15 @@ def load_media_map() -> MediaMap:
             logger().warning('Could not load media map: %s', e)
             return {}
 
-    return {
-        media_id: MediaHandler.build(**media_info)
-        for media_id, media_info in media_map.items()
-    }
+    parsed_map = {}
+    for media_id, media_info in media_map.items():
+        try:
+            parsed_map[media_id] = MediaHandler.build(**media_info)
+        except Exception as e:
+            logger().debug('Could not load media %s: %s', media_id, e)
+            continue
+
+    return parsed_map
 
 
 def save_media_map(new_map: MediaMap):
@@ -38,3 +43,12 @@ def save_media_map(new_map: MediaMap):
     with media_map_lock:
         redis = get_redis()
         redis.mset({MEDIA_MAP_VAR: json.dumps(new_map, cls=Message.Encoder)})
+
+
+def clear_media_map():
+    """
+    Clears the media map from the server.
+    """
+    with media_map_lock:
+        redis = get_redis()
+        redis.delete(MEDIA_MAP_VAR)