Make pydantic based config system

etc/smashbox.template.yaml

@@ -0,0 +1,116 @@
+# top directory where all local working files are kept (test working direcotires, test logs etc)
+smashdir: "~/smashdir"
+
+# name of the account used for testing; automatically picked if null
+oc_account_name: null
+
+# default number of users for tests involving multiple users (user number is appended to the oc_account_name)
+# this only applies to the tests involving multiple users
+oc_number_test_users: 3
+
+# name of the group used for testing
+oc_group_name: null
+
+# default number of groups for tests involving multiple groups (group number is appended to the oc_group_name)
+# this only applies to the tests involving multiple groups
+oc_number_test_groups: 1
+
+# password for test accounts: all test account will have the same password
+# if not set then it's an error
+oc_account_password: ""
+
+# owncloud test server
+# if left blank or "localhost" then the real hostname of the localhost will be set
+oc_server: ""
+
+# root of the owncloud installation as visible in the URL
+oc_root: "owncloud"
+
+# oc_webdav_endpoint will be computed based on oc_root
+
+# target folder on the server (this may not be compatible with all tests)
+oc_server_folder: ""
+
+# should we use protocols with SSL (https, ownclouds)
+oc_ssl_enabled: true
+
+# how to invoke shell commands on the server
+# for localhost there is no problem - leave it blank
+# for remote host it may be set like this: "ssh -t -l root $oc_server"
+# note: configure ssh for passwordless login 
+# note: -t option is to make it possible to run sudo
+oc_server_shell_cmd: ""
+
+# Data directory on the owncloud server.
+# computed based on oc_root as /var/www/html + oc_root + data
+
+# a path to server side tools (create_user.php, ...)  
+# it may be specified as relative path "dir" and then resolves to
+# <smashbox>/dir where <smashbox> is the top-level of of the tree
+# containing THIS configuration file
+oc_server_tools_path: "server-tools"
+
+# a path to ocsync command with options
+# this path should work for all client hosts
+#
+# it may be specified as relative path "./dir" and then resolves to
+# <smashbox>/dir where <smashbox> is the top-level of of the tree
+# containing THIS configuration file, e.g.: "./client/build/mirall/bin/owncloudcmd --trust"
+#
+# it may be specified as a basename executable (PATH will be used), e.g. "cernboxcmd --trust"
+#
+# it may be specified as absolute executable path, e.g. "/usr/bin/cernboxcmd --trust"
+#
+oc_sync_cmd: "cernboxcmd"
+
+# number of times to repeat ocsync run every time
+oc_sync_repeat: 1
+
+###########################################
+
+# unique identifier of your test run
+# if null then the runid is chosen automatically
+runid: null
+
+# if True then the local working directory path will have the runid added to it automatically
+workdir_runid_enabled: false
+
+# if True then the runid will be part of the oc_account_name automatically
+oc_account_runid_enabled: false
+
+####################################
+
+# this defines the default account cleanup procedure
+#   - "delete": delete account if exists and then create a new account with the same name
+#   - "keep": don't delete existing account but create one if needed
+#
+# these are not implemeted yet:
+#   - "sync_delete": delete all files via a sync run
+#   - "webdav_delete": delete all files via webdav DELETE request
+#   - "filesystem_delete": delete all files directly on the server's filesystem
+oc_account_reset_procedure: "delete"
+
+# this defined the default local run directory reset procedure
+#   - "delete": delete everything in the local run directory prior to running the test
+#   - "keep": keep all files (from the previous run)
+rundir_reset_procedure: "delete"
+
+web_user: "www-data"
+
+oc_admin_user: "at_admin"
+oc_admin_password: "admin"
+
+# Verbosity of curl client. 
+# If none then verbosity is on when smashbox run in --debug mode.
+# set it to True or False to override
+# This setting is no longer needed as pycurl isn't used
+# pycurl_verbose: null
+
+# scp port to be used in scp commands, used primarily when copying over the server log file
+scp_port: 22
+
+# user that can r+w the owncloud.log file (needs to be configured for passwordless login)
+oc_server_log_user: "www-data"
+
+# Reset the server log file and verify that no exceptions and other known errors have been logged
+oc_check_server_log: false

pyproject.toml

@@ -9,6 +9,8 @@ license = "AGPL-3.0"
 python = "^3.10"
 pytest = "^7.1.2"
 httpx = "^0.23.0"
+pydantic = "^1.9.1"
+PyYAML = "^6.0"
 
 [tool.poetry.dev-dependencies]
 black = "^22.3.0"

python/smashbox/__init__.py

@@ -0,0 +1 @@
+from . import webdav as webdav

python/smashbox/config.py

@@ -0,0 +1,99 @@
+import logging
+import os.path
+import pickle
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseSettings
+
+
+logger = None
+
+# this should probably be moved into a utilities module
+def get_logger(name: str = "config", level: int | None = None) -> logging.Logger:
+    global logger
+    if not logger:
+      if level is None:
+          level = logging.INFO  # change here to DEBUG if you want to debug config stuff
+      logging.basicConfig(level=level)
+    return logging.getLogger('.'.join(['smash', name]))
+
+
+class Configuration(BaseSettings):
+    """Root configuration object that parses the values defined in the config file."""
+    smashdir: str
+    oc_account_name: str | None
+    oc_number_test_users: int
+    oc_group_name: str | None
+    oc_number_test_groups: int
+    oc_account_password: str
+    oc_server: str
+    oc_root: str
+    oc_server_folder: str
+    oc_ssl_enabled: bool
+    oc_server_shell_cmd: str
+    oc_server_tools_path: str  # TODO: is this still needed?
+    oc_sync_cmd: str
+    oc_sync_repeat: int
+
+    runid: int | None
+    workdir_runid_enabled: bool
+    oc_account_runid_enabled: bool
+
+    oc_account_reset_procedure: Literal["delete", "keep"]  # there are some more types that are not yet implemented
+    rundir_reset_procedure: Literal["delete", "keep"]
+
+    web_user: str
+    oc_admin_user: str
+    oc_admin_password: str
+
+    scp_port: int
+    oc_server_log_user: str
+    oc_check_server_log: bool
+
+    @property
+    def oc_webdav_endpoint(self) -> str:
+        return os.path.join(self.oc_root, "remote.php/webdav")
+
+    @property
+    def oc_server_datadirectory(self) -> str:
+        return os.path.join("/var/www/html", self.oc_root, "data")
+
+    # these methods exists for backwards compatibility
+    def _dict(self, **args: Any) -> dict[str, object]:
+        """Returns a dictionary representation of the configuration object.
+        Any extra arguments passed are also returned in this dictionary."""
+        return {**self.dict(), **args}
+
+    def get(self, key: str, default: object) -> object:
+        """Returns the value of the specified setting, or the
+        default if the key doesn't exist."""
+        logger = get_logger()
+        logger.debug("config.get(%s,default=%s)", key, default)
+        return self._dict().get(key, default=default)
+
+
+def log_config(config: Configuration, level: int | None = None, hide_password: bool = False) -> None:
+    """Dump the entire configuration to the logging system at the given level.
+    If hide_password=True then do not show the real value of the options which contain "password" in their names.
+    """
+    logger = get_logger()
+    for key, val in config.dict().items():
+        if hide_password and "password" in key:
+            val = "***"
+        logger.log(level, "CONFIG: %s = %s", key, val)
+
+
+def load_config(fp: Path | str) -> Configuration:
+    """Loads and parses the specified configuration file."""
+    with open(fp, "r") as file:
+        return Configuration(**yaml.load(file))
+
+def configure_from_blob(fp: Path | str) -> Configuration:
+    return pickle.load(fp)
+
+def dump_config(config: Configuration, fp: Path | str) -> None:
+    """Serialize given config object as YAML and write it to the specified file."""
+    with open(fp, "w") as file:
+        yaml.dump(config.dict(), file)

python/smashbox/script.py

@@ -1,4 +1,3 @@
-
 import smashbox.compatibility.argparse as argparse
 
 def keyval_tuple(x):

python/smashbox/webdav.py

@@ -4,10 +4,10 @@
 import httpx
 
 from smashbox.utilities import *
-from smashbox.script import getLogger
+from smashbox.config import get_logger
 
 
-logger = getLogger()
+logger = get_logger()
 RequestType = Literal["MOVE", "PROPFIND", "PUT", "GET", "MKCOL", "DELTETE"]
 PropfindDepth = Literal["1", "0", "infinity"]
 OverwriteType = Literal["T", "F"]