diff --git a/etc/smashbox.template.yaml b/etc/smashbox.template.yaml new file mode 100644 index 0000000..21fc3aa --- /dev/null +++ b/etc/smashbox.template.yaml @@ -0,0 +1,116 @@ +# top directory where all local working files are kept (test working direcotires, test logs etc) +smashdir: "~/smashdir" + +# name of the account used for testing; automatically picked if null +oc_account_name: null + +# default number of users for tests involving multiple users (user number is appended to the oc_account_name) +# this only applies to the tests involving multiple users +oc_number_test_users: 3 + +# name of the group used for testing +oc_group_name: null + +# default number of groups for tests involving multiple groups (group number is appended to the oc_group_name) +# this only applies to the tests involving multiple groups +oc_number_test_groups: 1 + +# password for test accounts: all test account will have the same password +# if not set then it's an error +oc_account_password: "" + +# owncloud test server +# if left blank or "localhost" then the real hostname of the localhost will be set +oc_server: "" + +# root of the owncloud installation as visible in the URL +oc_root: "owncloud" + +# oc_webdav_endpoint will be computed based on oc_root + +# target folder on the server (this may not be compatible with all tests) +oc_server_folder: "" + +# should we use protocols with SSL (https, ownclouds) +oc_ssl_enabled: true + +# how to invoke shell commands on the server +# for localhost there is no problem - leave it blank +# for remote host it may be set like this: "ssh -t -l root $oc_server" +# note: configure ssh for passwordless login +# note: -t option is to make it possible to run sudo +oc_server_shell_cmd: "" + +# Data directory on the owncloud server. +# computed based on oc_root as /var/www/html + oc_root + data + +# a path to server side tools (create_user.php, ...) +# it may be specified as relative path "dir" and then resolves to +# /dir where is the top-level of of the tree +# containing THIS configuration file +oc_server_tools_path: "server-tools" + +# a path to ocsync command with options +# this path should work for all client hosts +# +# it may be specified as relative path "./dir" and then resolves to +# /dir where is the top-level of of the tree +# containing THIS configuration file, e.g.: "./client/build/mirall/bin/owncloudcmd --trust" +# +# it may be specified as a basename executable (PATH will be used), e.g. "cernboxcmd --trust" +# +# it may be specified as absolute executable path, e.g. "/usr/bin/cernboxcmd --trust" +# +oc_sync_cmd: "cernboxcmd" + +# number of times to repeat ocsync run every time +oc_sync_repeat: 1 + +########################################### + +# unique identifier of your test run +# if null then the runid is chosen automatically +runid: null + +# if True then the local working directory path will have the runid added to it automatically +workdir_runid_enabled: false + +# if True then the runid will be part of the oc_account_name automatically +oc_account_runid_enabled: false + +#################################### + +# this defines the default account cleanup procedure +# - "delete": delete account if exists and then create a new account with the same name +# - "keep": don't delete existing account but create one if needed +# +# these are not implemeted yet: +# - "sync_delete": delete all files via a sync run +# - "webdav_delete": delete all files via webdav DELETE request +# - "filesystem_delete": delete all files directly on the server's filesystem +oc_account_reset_procedure: "delete" + +# this defined the default local run directory reset procedure +# - "delete": delete everything in the local run directory prior to running the test +# - "keep": keep all files (from the previous run) +rundir_reset_procedure: "delete" + +web_user: "www-data" + +oc_admin_user: "at_admin" +oc_admin_password: "admin" + +# Verbosity of curl client. +# If none then verbosity is on when smashbox run in --debug mode. +# set it to True or False to override +# This setting is no longer needed as pycurl isn't used +# pycurl_verbose: null + +# scp port to be used in scp commands, used primarily when copying over the server log file +scp_port: 22 + +# user that can r+w the owncloud.log file (needs to be configured for passwordless login) +oc_server_log_user: "www-data" + +# Reset the server log file and verify that no exceptions and other known errors have been logged +oc_check_server_log: false diff --git a/poetry.lock b/poetry.lock index 0399885..68d4dc2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -206,6 +206,21 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "pydantic" +version = "1.9.1" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + [[package]] name = "pyparsing" version = "3.0.9" @@ -238,6 +253,14 @@ tomli = ">=1.0.0" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "rfc3986" version = "1.5.0" @@ -268,10 +291,18 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "typing-extensions" +version = "4.3.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "7a06c1b59e3c7d650ac73acc2cf7f09c7911d70107a314769dce9ab53df87a30" +content-hash = "e3392be6cf4d86dd2184ea6b7ab83d6e63ebdf93c4c708c0c45939268ef8ca96" [metadata.files] anyio = [ @@ -367,6 +398,7 @@ py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +pydantic = [] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -375,6 +407,7 @@ pytest = [ {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, ] +pyyaml = [] rfc3986 = [ {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, @@ -387,3 +420,4 @@ tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +typing-extensions = [] diff --git a/pyproject.toml b/pyproject.toml index 7692540..a9a6241 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,8 @@ license = "AGPL-3.0" python = "^3.10" pytest = "^7.1.2" httpx = "^0.23.0" +pydantic = "^1.9.1" +PyYAML = "^6.0" [tool.poetry.dev-dependencies] black = "^22.3.0" diff --git a/python/smashbox/__init__.py b/python/smashbox/__init__.py index e69de29..efcef1f 100644 --- a/python/smashbox/__init__.py +++ b/python/smashbox/__init__.py @@ -0,0 +1 @@ +from . import webdav as webdav diff --git a/python/smashbox/config.py b/python/smashbox/config.py new file mode 100644 index 0000000..35d192f --- /dev/null +++ b/python/smashbox/config.py @@ -0,0 +1,105 @@ +import logging +import os.path +import pickle +from pathlib import Path +from typing import Any, Literal + +import yaml +from pydantic import BaseSettings + + +# this should probably be moved into a utilities module +def get_logger(name: str = "config", level: int | None = None) -> logging.Logger: + if level is None: + level = ( + logging.INFO + ) # change here to DEBUG if you want to debug config stuff + logging.basicConfig(level=level) + return logging.getLogger(".".join(["smash", name])) + + +class Configuration(BaseSettings): + """Root configuration object that parses the values defined in the config file.""" + + smashdir: str + oc_account_name: str | None + oc_number_test_users: int + oc_group_name: str | None + oc_number_test_groups: int + oc_account_password: str + oc_server: str + oc_root: str + oc_server_folder: str + oc_ssl_enabled: bool + oc_server_shell_cmd: str + oc_server_tools_path: str # TODO: is this still needed? + oc_sync_cmd: str + oc_sync_repeat: int + + runid: int | None + workdir_runid_enabled: bool + oc_account_runid_enabled: bool + + oc_account_reset_procedure: Literal[ + "delete", "keep" + ] # there are some more types that are not yet implemented + rundir_reset_procedure: Literal["delete", "keep"] + + web_user: str + oc_admin_user: str + oc_admin_password: str + + scp_port: int + oc_server_log_user: str + oc_check_server_log: bool + + @property + def oc_webdav_endpoint(self) -> str: + return os.path.join(self.oc_root, "remote.php/webdav") + + @property + def oc_server_datadirectory(self) -> str: + return os.path.join("/var/www/html", self.oc_root, "data") + + # these methods exists for backwards compatibility + def _dict(self, **args: Any) -> dict[str, object]: + """Returns a dictionary representation of the configuration object. + Any extra arguments passed are also returned in this dictionary.""" + return {**self.dict(), **args} + + def get(self, key: str, default: object) -> object: + """Returns the value of the specified setting, or the + default if the key doesn't exist.""" + logger = get_logger() + logger.debug("config.get(%s,default=%s)", key, default) + return self._dict().get(key, default=default) + + +def log_config( + config: Configuration, level: int = logging.DEBUG, hide_password: bool = False +) -> None: + """Dump the entire configuration to the logging system at the given level. + If hide_password=True then do not show the real value of the options which contain "password" in their names. + """ + logger = get_logger() + for key, val in config.dict().items(): + if hide_password and "password" in key: + val = "***" + logger.log(level, "CONFIG: %s = %s", key, val) + + +def load_config(fp: Path | str) -> Configuration: + """Loads and parses the specified configuration file.""" + with open(fp, "r") as file: + return Configuration(**yaml.load(file, Loader=yaml.Loader)) + + +def configure_from_blob(fp: Path | str) -> Configuration: + with open(fp, "rb") as file: + return pickle.load(file) + + +def dump_config(config: Configuration, fp: Path | str) -> None: + """Serialize given config object as YAML and write it to the specified file.""" + with open(fp, "w") as file: + yaml.dump(config.dict(), file) diff --git a/python/smashbox/script.py b/python/smashbox/script.py index e1836ed..80e7719 100644 --- a/python/smashbox/script.py +++ b/python/smashbox/script.py @@ -1,130 +1,117 @@ +import logging +import os.path +from pathlib import Path +from typing import List, Sequence, Tuple import smashbox.compatibility.argparse as argparse +from smashbox.config import load_config, log_config, get_logger, Configuration -def keyval_tuple(x): - a,b = x.split('=',1) - return (a.strip(),b) -def arg_parser(**kwds): - """ Create an ArgumentParser with common options for smash scripts and tools. - """ - - parser = argparse.ArgumentParser(**kwds) - - parser.add_argument('--option', '-o', metavar="key=val", dest="options", type=keyval_tuple, action='append', help='set config option') - parser.add_argument('--dry-run', '-n', action='store_true', help='show config options and print what tests would be run') - parser.add_argument('--quiet', '-q', action="store_true", help='do not produce output (other than errors)') - parser.add_argument('--verbose', '-v', action="store_true", help='produce more output') - parser.add_argument('--debug', action="store_true", help='produce very verbose output') - parser.add_argument('--config','-c',dest="configs",default=[],action="append",help='config files (one or more), added on to of default config file') - return parser +main_config_file = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "etc", + "smashbox.conf.yaml", +) -import os.path -main_config_file = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))),'etc','smashbox.conf') - -class Configuration: - # you may use config object for string interpolation "..."%config - def __getitem__(self,x): - return getattr(self,x) - - def _dict(self,**args): - return dict(self.__dict__.items() + args.items()) - - def get(self,x,default): - logger = getLogger() - logger.debug('config.get(%s,default=%s)',repr(x),repr(default)) - try: - return getattr(self,x) - except AttributeError: - return default - - -config = Configuration() - -def configure_from_blob(config_blob): - import pickle - global config - config = pickle.loads(config_blob) - config_log(level=logging.DEBUG) - return config - -def dump_config_to_blob(): - import pickle - return pickle.dumps(config) - -def configure(cmdline_opts,config_files=None): - """ Initialize config object and return it. - - First exec the sequence of config_files (including the - main_config_file). All symbols defined by these files will be set - as attributes of the config object. - - Then process cmdline_opts (which is a list of tuples generated by - arg_parser). If attribute matching the option already exists (was - defined in a configuration file) then eval to the same type (if not - None). Otherwise leave string values. The string "None" is special - and it is always converted to None and may always be assigned. - - """ - - if config_files is None: - config_files = [] - - logger = getLogger() - - config_files = [main_config_file] + config_files - - for cf in config_files: - execfile(cf,{},config.__dict__) - - if cmdline_opts: - for key,val in cmdline_opts: - try: - if val == "None": - val = None - else: - attr = getattr(config,key) - # coerce val type to attr's type unless attr is set to None (then leave as-is ) - try: - if attr is not None: - val = type(attr)(val) - except ValueError,x: - # allow setting to None - logger.warning("cannot set option (type mismatch) %s=%s --> %s",key,repr(val),x) - continue - except AttributeError: - # if attr unknown then leave the val as-is (string) - pass - - setattr(config,key,val) - - config_log(level=logging.DEBUG) - - return config - -def config_log(level,hide_password=False): - """ Dump the entire configuration to the logging system at the given level. - If hide_password=True then do not show the real value of the options which contain "password" in their names. - """ - logger = getLogger() - for d in dir(config): - if not d.startswith("_") and d != "get": - value = repr(getattr(config,d)) - if hide_password and 'password' in d: - value = "***" - - logger.log(level,"CONFIG: %s = %s",d,value) +def keyval_tuple(x: str) -> Tuple[str, str]: + a, b = x.split("=", 1) + return (a.strip(), b) -import logging +def arg_parser(**kwds): + """Create an ArgumentParser with common options for smash scripts and tools.""" -logger = None -def getLogger(name="",level=None): - global logger - if not logger: - if level is None: - level = logging.INFO # change here to DEBUG if you want to debug config stuff - logging.basicConfig(level=level) + parser = argparse.ArgumentParser(**kwds) - return logging.getLogger('.'.join(['smash',name])) + parser.add_argument( + "--option", + "-o", + metavar="key=val", + dest="options", + type=keyval_tuple, + action="append", + help="set config option", + ) + parser.add_argument( + "--dry-run", + "-n", + action="store_true", + help="show config options and print what tests would be run", + ) + parser.add_argument( + "--quiet", + "-q", + action="store_true", + help="do not produce output (other than errors)", + ) + parser.add_argument( + "--verbose", "-v", action="store_true", help="produce more output" + ) + parser.add_argument( + "--debug", action="store_true", help="produce very verbose output" + ) + parser.add_argument( + "--config", + "-c", + dest="configs", + default=[], + action="append", + help="config files (one or more), added on to of default config file", + ) + return parser + + +def configure( + cmdline_opts: Sequence[Tuple[str, str]], + config_files: List[Path | str] | None = None, +) -> Configuration: + """Initialize config object and return it. + First read the sequence of config_files (including the + main_config_file). All symbols defined by these files will be set + as attributes of the config object. + Then process cmdline_opts (which is a list of tuples generated by + arg_parser). If attribute matching the option already exists (was + defined in a configuration file) then eval to the same type (if not + None). Otherwise leave string values. The string "None" is special + and it is always converted to None and may always be assigned. + """ + logger = get_logger() + config = load_config(main_config_file) + if config_files is None: + config_files = [] + + config_files = [main_config_file] + config_files + + # for cf in config_files: + # execfile(cf, {}, config.__dict__) + + if cmdline_opts: + for key, val in cmdline_opts: + try: + if val == "None": + val = None + else: + attr = getattr(config, key) + # coerce val type to attr's type unless attr is set to None (then leave as-is ) + try: + if attr is not None: + val = type(attr)(val) + except ValueError as x: + # allow setting to None + logger.warning( + "cannot set option (type mismatch) %s=%s --> %s", + key, + repr(val), + x, + ) + continue + except AttributeError: + # if attr unknown then leave the val as-is (string) + pass + + setattr(config, key, val) + + log_config(config, level=logging.DEBUG) + + return config \ No newline at end of file diff --git a/python/smashbox/webdav.py b/python/smashbox/webdav.py index aa31a1e..cef9c53 100644 --- a/python/smashbox/webdav.py +++ b/python/smashbox/webdav.py @@ -4,11 +4,11 @@ import httpx from smashbox.utilities import * -from smashbox.script import getLogger +from smashbox.config import get_logger -logger = getLogger() -RequestType = Literal["MOVE", "PROPFIND", "PUT", "GET", "MKCOL", "DELTETE"] +logger = get_logger() +RequestType = Literal["MOVE", "PROPFIND", "PUT", "GET", "MKCOL", "DELETE"] PropfindDepth = Literal["1", "0", "infinity"] OverwriteType = Literal["T", "F"]