Skip to content

Commit

Permalink
Merge branch 'config_refactor' into cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@bitbyt3r
bitbyt3r committed Apr 16, 2024
2 parents 6ae674b + 8a8c149 commit 3968cb1
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 373 deletions.
1 change: 1 addition & 0 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ Jinja2==3.1.3
paver==1.3.4
pip==24.0
psutil==5.9.8
pyyaml==6.0.1
redis==5.0.3
requests==2.31.0
rpctools==0.3.1
Expand Down
222 changes: 79 additions & 143 deletions sideboard/config.py
View file
Original file line number Diff line number Diff line change
@@ -1,92 +1,21 @@
from __future__ import unicode_literals
import pathlib
import json
import yaml
import os
import re

from os import unlink
from collections.abc import Sized, Iterable, Mapping
from copy import deepcopy
from tempfile import NamedTemporaryFile

import configobj
from validate import Validator


def uniquify(xs):
"""
Returns an order-preserved copy of `xs` with duplicate items removed.
>>> uniquify(['a', 'z', 'a', 'b', 'a', 'y', 'a', 'c', 'a', 'x'])
['a', 'z', 'b', 'y', 'c', 'x']
"""
is_listy = isinstance(xs, Sized) \
and isinstance(xs, Iterable) \
and not isinstance(xs, (Mapping, type(b''), type('')))
assert is_listy, 'uniquify requires a listy argument'

seen = set()
return [x for x in xs if x not in seen and not seen.add(x)]


class ConfigurationError(RuntimeError):
pass


def get_config_overrides():
"""
Returns a list of config file paths used to override the default config.
The SIDEBOARD_CONFIG_OVERRIDES environment variable may be set to a
semicolon separated list of absolute and/or relative paths. If the
SIDEBOARD_CONFIG_OVERRIDES is set, this function returns a list of its
contents, split on semicolons::
# SIDEBOARD_CONFIG_OVERRIDES='/absolute/config.ini;relative/config.ini'
return ['/absolute/config.ini', 'relative/config.ini']
If any of the paths listed in SIDEBOARD_CONFIG_OVERRIDES ends with the
suffix "<FILENAME>-defaults.<EXT>" then a similarly named path
"<FILENAME>.<EXT>" will also be included::
# SIDEBOARD_CONFIG_OVERRIDES='test-defaults.ini'
return ['test-defaults.ini', 'test.ini']
If the SIDEBOARD_CONFIG_OVERRIDES environment variable is NOT set, this
function returns a list with two relative paths::
return ['development-defaults.ini', 'development.ini']
"""
config_overrides = os.environ.get(
'SIDEBOARD_CONFIG_OVERRIDES',
'development-defaults.ini')

defaults_re = re.compile(r'(.+)-defaults(\.\w+)$')
config_paths = []
for config_path in uniquify([s.strip() for s in config_overrides.split(';')]):
config_paths.append(config_path)
m = defaults_re.match(config_path)
if m:
config_paths.append(m.group(1) + m.group(2))

return config_paths


def get_config_root():
"""
Returns the config root for the system, defaults to '/etc/sideboard'.
If the SIDEBOARD_CONFIG_ROOT environment variable is set, its contents
will be returned instead.
"""
default_root = '/etc/sideboard'
config_root = os.environ.get('SIDEBOARD_CONFIG_ROOT', default_root)
if config_root != default_root and not os.path.isdir(config_root):
raise AssertionError('cannot find {!r} directory'.format(config_root))
elif os.path.isdir(config_root) and not os.access(config_root, os.R_OK):
raise AssertionError('{!r} directory is not readable'.format(config_root))
return config_root


def get_module_and_root_dirs(requesting_file_path, is_plugin):
"""
Returns the "module_root" and "root" directories for the given file path.
Expand Down Expand Up @@ -114,63 +43,28 @@ def get_module_and_root_dirs(requesting_file_path, is_plugin):
Sideboard itself is making the request.
Returns:
tuple(str): The "module_root" and "root" directories for the
tuple(Path, Path, str): The "module_root" and "root" directories, and plugin name for the
given module.
"""
module_dir = os.path.dirname(os.path.abspath(requesting_file_path))
module_dir = pathlib.Path(requesting_file_path).parents[0]
if is_plugin:
from sideboard.lib import config
plugin_name = os.path.basename(module_dir)
root_dir = os.path.join(config['plugins_dir'], plugin_name)
if '_' in plugin_name and not os.path.exists(root_dir):
root_dir = os.path.join(config['plugins_dir'], plugin_name.replace('_', '-'))
plugin_name = module_dir.name
root_dir = pathlib.Path(config['plugins_dir']) / plugin_name
if '_' in plugin_name and not root_dir.exists():
root_dir = pathlib.Path(config['plugins_dir']) / plugin_name.replace('_', '-')
else:
root_dir = os.path.realpath(os.path.join(module_dir, '..'))
return module_dir, root_dir
root_dir = module_dir.parents[0]
plugin_name = "sideboard"
return module_dir, root_dir, plugin_name


def get_config_files(requesting_file_path, is_plugin):
"""
Returns a list of absolute paths to config files for the given file path.
When the returned config files are parsed by ConfigObj each subsequent
file will override values in earlier files.
If `is_plugin` is `True` the first of the returned files is:
* /etc/sideboard/plugins.d/<PLUGIN_NAME>.cfg, which is the config file we
expect in production
If `is_plugin` is `False` the first two returned files are:
* /etc/sideboard/sideboard-core.cfg, which is the sideboard core config
file we expect in production
* /etc/sideboard/sideboard-server.cfg, which is the sideboard server config
file we expect in production
The rest of the files returned are as follows, though we wouldn't
necessarily expect these to exist on a production install (these are
controlled by SIDEBOARD_CONFIG_OVERRIDES):
* <PROJECT_DIR>/development-defaults.ini, which can be checked into source
control and include whatever we want to be present in a development
environment.
* <PROJECT_DIR>/development.ini, which shouldn't be checked into source
control, allowing a developer to include local settings not shared with
others.
When developing on a machine with an installed production config file, we
might want to ignore the "real" config file and limit ourselves to only the
development files. This behavior is turned on by setting the environment
variable SIDEBOARD_MODULE_TESTING to any value. (This environment variable
is also used elsewhere to turn off automatically loading all plugins in
order to facilitate testing modules which rely on Sideboard but which are
not themselves Sideboard plugins.)
If the file is in a plugin we check the environment variable
<PLUGIN NAME>_CONFIG_FILES and return any paths from there, seperated by ;
Args:
requesting_file_path (str): The Python __file__ of the module
Expand All @@ -182,24 +76,56 @@ def get_config_files(requesting_file_path, is_plugin):
Returns:
list(str): List of absolute paths to config files for the given module.
"""
config_root = get_config_root()
module_dir, root_dir = get_module_and_root_dirs(requesting_file_path, is_plugin)
module_name = os.path.basename(module_dir)

if 'SIDEBOARD_MODULE_TESTING' in os.environ:
base_configs = []
elif is_plugin:
base_configs = [os.path.join(config_root, 'plugins.d', '{}.cfg'.format(module_name.replace('_', '-')))]
else:
assert module_name == 'sideboard', 'Unexpected module name {!r} requesting "non-plugin" configuration files'.format(module_name)
base_configs = [
os.path.join(config_root, 'sideboard-core.cfg'),
os.path.join(config_root, 'sideboard-server.cfg')
]

config_overrides = [os.path.join(root_dir, config_path) for config_path in get_config_overrides()]
return base_configs + config_overrides

module_dir, root_dir, plugin_name = get_module_and_root_dirs(requesting_file_path, is_plugin)
config_files_str = os.environ.get(f"{plugin_name.upper()}_CONFIG_FILES", "")
absolute_config_files = []
if config_files_str:
config_files = [pathlib.Path(x) for x in config_files_str.split(";")]
for path in config_files:
if path.is_absolute():
if not path.exists():
raise ValueError(f"Config file {path} specified in {plugin_name.upper()}_CONFIG_FILES does not exist!")
absolute_config_files.append(path)
else:
if not (root_dir / path).exists():
raise ValueError(f"Config file {root_dir / path} specified in {plugin_name.upper()}_CONFIG_FILES does not exist!")
absolute_config_files.append(root_dir / path)
return absolute_config_files

def normalize_name(name):
return name.replace(".", "_")

def load_section_from_environment(path, section):
"""
Looks for configuration in environment variables.
Args:
path (str): The prefix of the current config section. For example,
sideboard.ini:
[cherrypy]
server.thread_pool: 10
would translate to sideboard_cherrypy_server.thread_pool
section (configobj.ConfigObj): The section of the configspec to search
for the current path in.
"""
config = {}
for setting in section:
if setting == "__many__":
prefix = f"{path}_"
for envvar in os.environ:
if envvar.startswith(prefix) and not envvar.split(prefix, 1)[1] in [normalize_name(x) for x in section]:
config[envvar.split(prefix, 1)[1]] = os.environ[envvar]
else:
if isinstance(section[setting], configobj.Section):
child_path = f"{path}_{setting}"
child = load_section_from_environment(child_path, section[setting])
if child:
config[setting] = child
else:
name = normalize_name(f"{path}_{setting}")
if name in os.environ:
config[setting] = yaml.safe_load(os.environ.get(normalize_name(name)))
return config

def parse_config(requesting_file_path, is_plugin=True):
"""
Expand All @@ -225,18 +151,28 @@ def parse_config(requesting_file_path, is_plugin=True):
Returns:
ConfigObj: The resulting configuration object.
"""
module_dir, root_dir = get_module_and_root_dirs(requesting_file_path, is_plugin)
module_dir, root_dir, plugin_name = get_module_and_root_dirs(requesting_file_path, is_plugin)

specfile = os.path.join(module_dir, 'configspec.ini')
spec = configobj.ConfigObj(specfile, interpolation=False, list_values=False, encoding='utf-8', _inspec=True)
specfile = module_dir / 'configspec.ini'
spec = configobj.ConfigObj(str(specfile), interpolation=False, list_values=False, encoding='utf-8', _inspec=True)

# to allow more/better interpolations
root_conf = ['root = "{}"\n'.format(root_dir), 'module_root = "{}"\n'.format(module_dir)]
temp_config = configobj.ConfigObj(root_conf, interpolation=False, encoding='utf-8')

for config_path in get_config_files(requesting_file_path, is_plugin):
# this gracefully handles nonexistent files
temp_config.merge(configobj.ConfigObj(config_path, encoding='utf-8', interpolation=False))
file_config = configobj.ConfigObj(str(config_path), encoding='utf-8', interpolation=False)
if os.environ.get("LOG_CONFIG", "false").lower() == "true":
print(f"File config for {plugin_name} from {config_path}")
print(json.dumps(file_config, indent=2, sort_keys=True))
temp_config.merge(file_config)

environment_config = load_section_from_environment(plugin_name, spec)
if os.environ.get("LOG_CONFIG", "false").lower() == "true":
print(f"Environment config for {plugin_name}")
print(json.dumps(environment_config, indent=2, sort_keys=True))
temp_config.merge(configobj.ConfigObj(environment_config, encoding='utf-8', interpolation=False))

# combining the merge files to one file helps configspecs with interpolation
with NamedTemporaryFile(delete=False) as config_outfile:
Expand Down
18 changes: 17 additions & 1 deletion sideboard/configspec.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,15 +118,27 @@ profiling.strip_dirs = boolean(default=False)

server.socket_host = string(default="127.0.0.1")
server.socket_port = integer(default=8282)
server.thread_pool = integer(default=10)

tools.reset_threadlocal.on = boolean(default=True)

tools.sessions.on = boolean(default=True)
tools.sessions.path = string(default="/")
tools.sessions.timeout = integer(default=30)
tools.sessions.timeout = integer(default=60)
tools.sessions.storage_type = string(default="file")
tools.sessions.storage_path = string(default="%(root)s/data/sessions")
tools.sessions.secure = boolean(default=False)
tools.sessions.prefix = string(default=sideboard)

# RedisSession specific values
tools.sessions.host = string(default="127.0.0.1")
tools.sessions.port = integer(default=6379)
tools.sessions.db = integer(default=0)
tools.sessions.password = string(default=None)
tools.sessions.tls_skip_verify = boolean(default=False)
tools.sessions.is_sentinel = boolean(default=False)
tools.sessions.ssl = boolean(default=False)
tools.sessions.user = string(default="")

# Built-in CherryPy web server stats page
tools.cpstats.on = boolean(default=False)
Expand All @@ -145,6 +157,10 @@ cherrypy.access = option("TRACE", "DEBUG", "INFO", "WARNING", "WARN", "ERROR", "
__many__ = option("TRACE", "DEBUG", "INFO", "WARN", "WARNING", "ERROR", "CRITICAL", default="INFO")

[handlers]
[[stdout]]
class = string(default="logging.StreamHandler")
stream = string(default="ext://sys.stdout")
formatter = string(default="indent_multiline")
[[__many__]]
formatter = string(default="default")
___many___ = string()
Expand Down
51 changes: 23 additions & 28 deletions sideboard/internal/logging.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
from __future__ import unicode_literals, absolute_import
import os
import logging.config

from sideboard.config import config, get_config_root
from sideboard.config import config


class IndentMultilinesLogFormatter(logging.Formatter):
Expand All @@ -18,29 +17,25 @@ def format(self, record):


def _configure_logging():
fname = os.path.join(get_config_root(), 'logging.cfg')
if os.path.exists(fname):
logging.config.fileConfig(fname, disable_existing_loggers=True)
else:
# ConfigObj doesn't support interpolation escaping, so we manually work around it here
formatters = config['formatters'].dict()
for formatter in formatters.values():
formatter['format'] = formatter['format'].replace('$$', '%')
formatter['datefmt'] = formatter['datefmt'].replace('$$', '%') or None
formatters['indent_multiline'] = {
'()': IndentMultilinesLogFormatter,
'format': formatters['default']['format']
}
logging.config.dictConfig({
'version': 1,
'root': {
'level': config['loggers']['root'],
'handlers': config['handlers'].dict().keys()
},
'loggers': {
name: {'level': level}
for name, level in config['loggers'].items() if name != 'root'
},
'handlers': config['handlers'].dict(),
'formatters': formatters
})
# ConfigObj doesn't support interpolation escaping, so we manually work around it here
formatters = config['formatters'].dict()
for formatter in formatters.values():
formatter['format'] = formatter['format'].replace('$$', '%')
formatter['datefmt'] = formatter['datefmt'].replace('$$', '%') or None
formatters['indent_multiline'] = {
'()': IndentMultilinesLogFormatter,
'format': formatters['default']['format']
}
logging.config.dictConfig({
'version': 1,
'root': {
'level': config['loggers']['root'],
'handlers': config['handlers'].dict().keys()
},
'loggers': {
name: {'level': level}
for name, level in config['loggers'].items() if name != 'root'
},
'handlers': config['handlers'].dict(),
'formatters': formatters
})
Loading

0 comments on commit 3968cb1

Please sign in to comment.