diff --git a/doc/conf.py b/doc/conf.py index c5bcbf1288..a547e932d1 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -12,7 +12,12 @@ # import os import sys -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +import textwrap + +TOPDIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(TOPDIR, "..")) + +import osc.conf # -- Project information ----------------------------------------------------- @@ -52,6 +57,23 @@ master_doc = 'index' +# -- Generate documents ------------------------------------------------- + +osc.conf._model_to_rst( + cls=osc.conf.Options, + title="Configuration file", + description=textwrap.dedent( + """ + The configuration file path is ``$XDG_CONFIG_HOME/osc/oscrc``, which usually translates into ``~/.config/osc/oscrc``. + """ + ), + sections={ + "Host options": osc.conf.HostOptions, + }, + output_file=os.path.join(TOPDIR, "oscrc.rst"), +) + + # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -64,3 +86,11 @@ # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] + + +# -- Options for MAN output ------------------------------------------------- + +# (source start file, name, description, authors, manual section). +man_pages = [ + ("oscrc", "oscrc", "openSUSE Commander configuration file", "openSUSE project ", 5), +] diff --git a/doc/index.rst b/doc/index.rst index be6c57742f..8e49867471 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -21,6 +21,7 @@ API: api/modules plugins/index + oscrc diff --git a/osc/commandline.py b/osc/commandline.py index 0894aa2b49..1000db0e42 100644 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -1078,10 +1078,7 @@ def post_argparse(self): except oscerr.NoConfigfile as e: print(e.msg, file=sys.stderr) print('Creating osc configuration file %s ...' % e.file, file=sys.stderr) - apiurl = conf.DEFAULTS['apiurl'] - if self.options.apiurl: - apiurl = self.options.apiurl - conf.interactive_config_setup(e.file, apiurl) + conf.interactive_config_setup(e.file, self.options.apiurl) print('done', file=sys.stderr) self.post_argparse() except oscerr.ConfigMissingApiurl as e: diff --git a/osc/conf.py b/osc/conf.py index b1a4cd5067..c50bd05fc8 100644 --- a/osc/conf.py +++ b/osc/conf.py @@ -36,18 +36,25 @@ """ +import collections import errno import getpass +import http.client import os import re +import shutil import sys +import textwrap +from io import BytesIO from io import StringIO from urllib.parse import urlsplit from . import credentials from . import OscConfigParser from . import oscerr +from .util import xdg from .util.helper import raw_input +from .util.models import * GENERIC_KEYRING = False @@ -59,384 +66,1201 @@ pass -def _get_processors(): - """ - get number of processors (online) based on - SC_NPROCESSORS_ONLN (returns 1 if config name/os.sysconf does not exist). - """ - try: - return os.sysconf('SC_NPROCESSORS_ONLN') - except (AttributeError, ValueError): - return 1 +class Password(collections.UserString): + def __init__(self, data): + self._data = data + @property + def data(self): + if callable(self._data): + # if ``data`` is a function, call it every time the string gets evaluated + # we use the password only from time to time to make a session cookie + # and there's no need to keep the password in program memory longer than necessary + result = self._data() + if result is None: + raise oscerr.OscIOError(None, "Unable to retrieve password") + return self._data -def _identify_osccookiejar(): - if os.path.isfile(os.path.join(os.path.expanduser("~"), '.osc_cookiejar')): - # For backwards compatibility, use the old location if it exists - return '~/.osc_cookiejar' + def __format__(self, format_spec): + if format_spec.endswith("s"): + return f"{self.__str__():{format_spec}}" + return super().__format__(format_spec) - if os.getenv('XDG_STATE_HOME', '') != '': - osc_state_dir = os.path.join(os.getenv('XDG_STATE_HOME'), 'osc') - else: - osc_state_dir = os.path.join(os.path.expanduser("~"), '.local', 'state', 'osc') - - return os.path.join(osc_state_dir, 'cookiejar') - - -DEFAULTS = {'apiurl': 'https://api.opensuse.org', - 'user': None, - 'pass': None, - 'passx': None, - 'sshkey': None, - 'packagecachedir': '/var/tmp/osbuild-packagecache', - 'su-wrapper': 'sudo', - - # build type settings - 'build-cmd': '/usr/bin/build', - 'build-type': '', # may be empty for chroot, kvm or xen - 'build-root': '/var/tmp/build-root/%(repo)s-%(arch)s', - 'build-uid': '', # use the default provided by build - 'build-device': '', # required for VM builds - 'build-memory': '0', # required for VM builds - 'build-shell-after-fail': '0', # optional for VM builds - 'build-swap': '', # optional for VM builds - 'build-vmdisk-rootsize': '0', # optional for VM builds - 'build-vmdisk-swapsize': '0', # optional for VM builds - 'build-vmdisk-filesystem': '', # optional for VM builds - 'build-vm-user': '', # optional for VM builds - 'build-kernel': '', # optional for VM builds - 'build-initrd': '', # optional for VM builds - 'download-assets-cmd': '/usr/lib/build/download_assets', # optional for scm/git based builds - - 'build-jobs': str(_get_processors()), - 'builtin_signature_check': '1', # by default use builtin check for verify pkgs - 'icecream': '0', - 'ccache': '0', - 'sccache': '0', - 'sccache_uri': '', - - 'buildlog_strip_time': '0', # strips the build time from the build log - - 'debug': '0', - 'http_debug': '0', - 'http_full_debug': '0', - 'http_retries': '3', - 'verbose': '0', - 'no_preinstallimage': '0', - 'traceback': '0', - 'post_mortem': '0', - 'use_keyring': '0', - 'cookiejar': _identify_osccookiejar(), - # fallback for osc build option --no-verify - 'no_verify': '0', - - # Disable hdrmd5 checks of downloaded and cached packages in `osc build` - # Recommended value: 0 - # - # OBS builds the noarch packages once per binary arch. - # Such noarch packages are supposed to be nearly identical across all build arches, - # any discrepancy in the payload and dependencies is considered a packaging bug. - # But to guarantee that the local builds work identically to builds in OBS, - # using the arch-specific copy of the noarch package is required. - # Unfortunatelly only one of the noarch packages gets distributed - # and can be downloaded from a local mirror. - # All other noarch packages are available through the OBS API only. - # Since there is currently no information about hdrmd5 checksums of published noarch packages, - # we download them, verify hdrmd5 and re-download the package from OBS API on mismatch. - # - # The same can also happen for architecture depend packages when someone is messing around - # with the source history or the release number handling in a way that it is not increasing. - # - # If you want to save some bandwidth and don't care about the exact rebuilds - # you can turn this option on to disable hdrmd5 checks completely. - 'disable_hdrmd5_check': '0', - - # enable project tracking by default - 'do_package_tracking': '1', - # default for osc build - 'extra-pkgs': '', - # default repository - 'build_repository': 'openSUSE_Factory', - # default project for branch or bco - 'getpac_default_project': 'openSUSE:Factory', - # alternate filesystem layout: have multiple subdirs, where colons were. - 'checkout_no_colon': '0', - # project separator - 'project_separator': ':', - # change filesystem layout: avoid checkout from within a proj or package dir. - 'checkout_rooted': '0', - # local files to ignore with status, addremove, .... - 'exclude_glob': '.osc CVS .svn .* _linkerror *~ #*# *.orig *.bak *.changes.vctmp.*', - # whether to print Web UI links to directly insert in browser (where possible) - 'print_web_links': '0', - # limit the age of requests shown with 'osc req list'. - # this is a default only, can be overridden by 'osc req list -D NNN' - # Use 0 for unlimted. - 'request_list_days': 0, - # check for unversioned/removed files before commit - 'check_filelist': '1', - # check for pending requests after executing an action (e.g. checkout, update, commit) - 'check_for_request_on_action': '1', - # what to do with the source package if the submitrequest has been accepted - 'submitrequest_on_accept_action': '', - 'request_show_interactive': '0', - 'request_show_source_buildstatus': '0', - # if a review is accepted in interactive mode and a group - # was specified the review will be accepted for this group - 'review_inherit_group': '0', - 'submitrequest_accepted_template': '', - 'submitrequest_declined_template': '', - 'linkcontrol': '0', - 'include_request_from_project': '1', - 'local_service_run': '1', - "exclude_files": "", - "include_files": "", - - # Maintenance defaults to OBS instance defaults - 'maintained_attribute': 'OBS:Maintained', - 'maintenance_attribute': 'OBS:MaintenanceProject', - 'maintained_update_project_attribute': 'OBS:UpdateProject', - 'show_download_progress': '0', - # path to the vc script - 'vc-cmd': '/usr/lib/build/vc', - - # heuristic to speedup Package.status - 'status_mtime_heuristic': '0' - } - -# some distros like Debian rename and move build to obs-build -if not os.path.isfile('/usr/bin/build') and os.path.isfile('/usr/bin/obs-build'): - DEFAULTS['build-cmd'] = '/usr/bin/obs-build' -if not os.path.isfile('/usr/lib/build/vc') and os.path.isfile('/usr/lib/obs-build/vc'): - DEFAULTS['vc-cmd'] = '/usr/lib/obs-build/vc' - -api_host_options = ['user', 'pass', 'passx', 'aliases', 'http_headers', 'realname', 'email', 'sslcertck', 'cafile', 'capath', 'trusted_prj', - 'downloadurl', 'sshkey', 'disable_hdrmd5_check'] - - -# _integer_opts and _boolean_opts specify option types for both global options as well as api_host_options -_integer_opts = ("build-jobs", "build-memory", "build-vmdisk-rootsize", "build-vmdisk-swapsize", "http_retries", "icecream", "request_list_days") - -_boolean_opts = ( - 'debug', 'do_package_tracking', 'http_debug', 'post_mortem', 'traceback', 'check_filelist', - 'checkout_no_colon', 'checkout_rooted', 'check_for_request_on_action', 'linkcontrol', 'show_download_progress', 'request_show_interactive', - 'request_show_source_buildstatus', 'review_inherit_group', 'use_keyring', 'no_verify', 'disable_hdrmd5_check', 'builtin_signature_check', - 'http_full_debug', 'include_request_from_project', 'local_service_run', 'buildlog_strip_time', 'no_preinstallimage', - 'status_mtime_heuristic', 'print_web_links', 'ccache', 'sccache', 'build-shell-after-fail', 'allow_http', 'sslcertck', ) - - -def apply_option_types(config, conffile=""): - """ - Return a copy of `config` dictionary with values converted to their expected types - according to the enumerated option types (_boolean_opts, _integer_opts). - """ - config = config.copy() - cp = OscConfigParser.OscConfigParser(config) - cp.add_section("general") +HttpHeader = NewType("HttpHeader", Tuple[str, str]) - typed_opts = ((_boolean_opts, cp.getboolean, bool), (_integer_opts, cp.getint, int)) - for opts, meth, typ in typed_opts: - for opt in opts: - if opt not in config: - continue - if isinstance(config[opt], typ): - continue - try: - config[opt] = meth('general', opt) - except ValueError as e: - msg = 'cannot parse \'%s\' setting: %s' % (opt, str(e)) - raise oscerr.ConfigError(msg, conffile) - return config +class OscOptions(BaseModel): + # compat function with the config dict + def _get_field_name(self, name): + if name in self.__fields__: + return name + for field_name, field in self.__fields__.items(): + ini_key = field.extra.get("ini_key", None) + if ini_key == name: + return field_name -# being global to this module, this dict can be accessed from outside -# it will hold the parsed configuration -config = DEFAULTS.copy() -config = apply_option_types(config) + return None + # compat function with the config dict + def __getitem__(self, name): + field_name = self._get_field_name(name) + if field_name is None: + field_name = name + try: + return getattr(self, field_name) + except AttributeError: + raise KeyError(name) + + # compat function with the config dict + def __setitem__(self, name, value): + field_name = self._get_field_name(name) + if field_name is None: + field_name = name + setattr(self, field_name, value) + + # compat function with the config dict + def __contains__(self, name): + try: + self[name] + except KeyError: + return False + return True + + # compat function with the config dict + def setdefault(self, name, default=None): + field_name = self._get_field_name(name) + # we're ignoring ``default`` because the field always exists + return getattr(self, field_name, None) + + # compat function with the config dict + def get(self, name, default=None): + try: + return self[name] + except KeyError: + return default + + def set_value_from_string(self, name, value): + field_name = self._get_field_name(name) + field = self.__fields__[field_name] + + if not isinstance(value, str): + setattr(self, field_name, value) + return + + if not value.strip(): + if field.is_optional: + setattr(self, field_name, value) + return + + if field.origin_type is Password: + value = Password(value) + setattr(self, field_name, value) + return + + if field.type is List[HttpHeader]: + value = http.client.parse_headers(BytesIO(value.strip().encode("utf-8"))).items() + setattr(self, field_name, value) + return + + if field.origin_type is list: + # split list options into actual lists + value = re.split(r"[, ]+", value) + setattr(self, field_name, value) + return + + if field.origin_type is bool: + if value.lower() in ["1", "yes", "true", "on"]: + value = True + setattr(self, field_name, value) + return + if value.lower() in ["0", "no", "false", "off"]: + value = False + setattr(self, field_name, value) + return + + if field.origin_type is int: + value = int(value) + setattr(self, field_name, value) + return + + setattr(self, field_name, value) + + +class HostOptions(OscOptions): + apiurl: str = Field( + default=None, + description=textwrap.dedent( + """ + URL to the API server. + """ + ), + ) # type: ignore[assignment] + + aliases: List[str] = Field( + default=[], + ini_type="space-separated-list", + ) # type: ignore[assignment] + + username: str = Field( + default=None, + description=textwrap.dedent( + """ + """ + ), + ini_key="user", + ) # type: ignore[assignment] + + password: Optional[Password] = Field( + default=None, + description=textwrap.dedent( + """ + """ + ), + json_schema_extra={ + "ini_key": "pass", + }, + ini_key="pass", + ) # type: ignore[assignment] + + sshkey: Optional[str] = Field( + default=None, + ) # type: ignore[assignment] + + downloadurl: Optional[str] = Field( + default=None, + ) # type: ignore[assignment] + + sslcertck: bool = Field( + default=True, + ) # type: ignore[assignment] + + trusted_prj: List[str] = Field( + default=[], + ini_type="space-separated-list", + ) # type: ignore[assignment] + + credentials_mgr_class: Optional[str] = Field( + default=None, + ) # type: ignore[assignment] + + http_headers: List[HttpHeader] = Field( + default=[], + ini_type="newline-separated-list", + ) # type: ignore[assignment] + + disable_hdrmd5_check: bool = Field( + default=False, + ) # type: ignore[assignment] + + allow_http: bool = Field( + default=False, + ) # type: ignore[assignment] + + passx: Optional[str] = Field(default=None) # type: ignore[assignment] + realname: Optional[str] = Field(default=None) # type: ignore[assignment] + email: Optional[str] = Field(default=None) # type: ignore[assignment] + cafile: Optional[str] = Field(default=None) # type: ignore[assignment] + capath: Optional[str] = Field(default=None) # type: ignore[assignment] + + +class Options(OscOptions): + # for internal use + conffile: Optional[str] = Field( + default=None, + exclude=True, + ) # type: ignore[assignment] + + @property + def apiurl_aliases(self): + result = {} + for apiurl, opts in self.api_host_options.items(): + result[apiurl] = apiurl + for alias in opts.aliases: + result[alias] = apiurl + return result + + # OPTIONS + section_generic: str = Field( + default="Generic options", + exclude=True, + section=True, + ) # type: ignore[assignment] + + apiurl: str = Field( + default="https://api.opensuse.org", + description=textwrap.dedent( + """ + Default URL to the API server. + Credentials and other ``apiurl`` specific settings must be configured + in a ``[$apiurl]`` config section or via API in a ``host_options`` entry. + """ + ), + ) # type: ignore[assignment] + + username: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + DEPRECATED: Specify user for each apiurl instead. + """ + ), + ini_key="user", + ) # type: ignore[assignment] + + password: Optional[Password] = Field( + default=None, + description=textwrap.dedent( + """ + DEPRECATED: Specify password for each apiurl instead. + """ + ), + ini_key="pass", + ) # type: ignore[assignment] + + passx: Optional[str] = Field( + default=None, + ) # type: ignore[assignment] + + api_host_options: Dict[str, HostOptions] = Field( + default={}, + ini_skip=True, + ) # type: ignore[assignment] + + # AUTHENTICATION OPTIONS + section_auth: str = Field( + default="Authentication options", + exclude=True, + section=True, + ) # type: ignore[assignment] + + sshkey: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + A pointer to public SSH key that corresponds with a private SSH used for authentication: + + - path to the public SSH key + - public SSH key filename (must be placed in ~/.ssh) + + NOTE: The private key may not be available on disk because it could be in a GPG keyring, on YubiKey or forwarded through SSH agent. + """ + ), + ) # type: ignore[assignment] + + use_keyring: bool = Field( + default=False, + description=textwrap.dedent( + """ + Enable keyring as an option for storing passwords. + """ + ), + ) # type: ignore[assignment] + + # VERBOSITY OPTIONS + section_verbosity: str = Field( + default="Verbosity options", + exclude=True, + section=True, + ) # type: ignore[assignment] + + verbose: bool = Field( + default=False, + description=textwrap.dedent( + """ + Increase amount of printed information to stdout. + """ + ), + ) # type: ignore[assignment] + + debug: bool = Field( + default=False, + description=textwrap.dedent( + """ + Print debug information to stderr. + """ + ), + ) # type: ignore[assignment] + + http_debug: bool = Field( + default=False, + description=textwrap.dedent( + """ + Print HTTP traffic to stderr. + """ + ), + ) # type: ignore[assignment] + + http_full_debug: bool = Field( + default=False, + description=textwrap.dedent( + """ + [CAUTION!] Print HTTP traffic incl. authentication data to stderr. + """ + ), + ) # type: ignore[assignment] + + post_mortem: bool = Field( + default=False, + description=textwrap.dedent( + """ + Jump into a debugger when an unandled exception occurs. + """ + ), + ) # type: ignore[assignment] + + traceback: bool = Field( + default=False, + description=textwrap.dedent( + """ + Print full traceback to stderr when an unandled exception occurs. + """ + ), + ) # type: ignore[assignment] + + show_download_progress: bool = Field( + default=True, + description=textwrap.dedent( + """ + Show download progressbar. + """ + ), + ) # type: ignore[assignment] + + # CONNECTION OPTIONS + section_connection: str = Field( + default="Connection options", + exclude=True, + section=True, + ) # type: ignore[assignment] + + http_retries: int = Field( + default=3, + description=textwrap.dedent( + """ + Number of retries on HTTP error. + """ + ), + ) # type: ignore[assignment] + + cookiejar: str = Field( + default=os.path.join(xdg.XDG_STATE_HOME, "osc", "cookiejar"), + description=textwrap.dedent( + """ + Path to a cookie jar that stores session cookies. + """ + ), + ) # type: ignore[assignment] + + # SCM OPTIONS + section_scm: str = Field( + default="SCM options", + exclude=True, + section=True, + ) # type: ignore[assignment] + + local_service_run: bool = Field( + default=True, + description=textwrap.dedent( + """ + Run local services during commit. + """ + ), + ) # type: ignore[assignment] + + getpac_default_project: str = Field( + default="openSUSE:Factory", + description=textwrap.dedent( + """ + The default project for ``osc getpac`` and ``osc bco``. + The value is a space separated list of strings. + """ + ), + ) # type: ignore[assignment] + + exclude_glob: List[str] = Field( + default=[".osc", "CVS", ".svn", ".*", "_linkerror", "*~", "#*#", "*.orig", "*.bak", "*.changes.vctmp.*"], + description=textwrap.dedent( + """ + Space separated list of files ignored by SCM. + The files can contain globs. + """ + ), + ini_type="space-separated-list", + ) # type: ignore[assignment] + + exclude_files: List[str] = Field( + default=[], + description=textwrap.dedent( + """ + Files that match the listed glob patterns get skipped during checkout. + """ + ), + ) # type: ignore[assignment] + + include_files: List[str] = Field( + default=[], + description=textwrap.dedent( + """ + Files that do not match the listed glob patterns get skipped during checkout. + The ``exclude_files`` option takes priority over ``include_files``. + """ + ), + ) # type: ignore[assignment] + + checkout_no_colon: bool = Field( + default=False, + description=textwrap.dedent( + """ + Use '/' as project separator instead the default ':' and create corresponding subdirs. + If enabled, it takes priority over the ``project_separator`` option. + """ + ), + ) # type: ignore[assignment] + + project_separator: str = Field( + default=":", + description=textwrap.dedent( + """ + Use the specified string to separate projects. + """ + ), + ) # type: ignore[assignment] + + check_filelist: bool = Field( + default=True, + description=textwrap.dedent( + """ + Check for untracked files and removed files before commit. + """ + ), + ) # type: ignore[assignment] + + do_package_tracking: bool = Field( + default=True, + description=textwrap.dedent( + """ + Track packages in parent project's .osc/_packages. + """ + ), + ) # type: ignore[assignment] + + checkout_rooted: bool = Field( + default=False, + description=textwrap.dedent( + """ + Prevent checking out projects inside other projects or packages. + """ + ), + ) # type: ignore[assignment] + + status_mtime_heuristic: bool = Field( + default=False, + description=textwrap.dedent( + """ + Consider a file with a modified mtime as modified. + """ + ), + ) # type: ignore[assignment] + + linkcontrol: bool = Field( + default=False, + description=textwrap.dedent( + # TODO: explain what linkcontrol does + """ + """ + ), + ) # type: ignore[assignment] + + # BUILD OPTIONS + section_build: str = Field( + default="Build options", + exclude=True, + section=True, + ) # type: ignore[assignment] + + build_repository: str = Field( + default="openSUSE_Factory", + description=textwrap.dedent( + """ + The default repository used when the ``repository`` argument is omitted from ``osc build``. + """ + ), + ) # type: ignore[assignment] + + buildlog_strip_time: bool = Field( + default=False, + description=textwrap.dedent( + """ + Strip the build time from the build logs. + """ + ), + ) # type: ignore[assignment] + + package_cache_dir: str = Field( + default="/var/tmp/osbuild-packagecache", + description=textwrap.dedent( + """ + The directory where downloaded packages are stored. Must be writable by you. + """ + ), + ini_key="packagecachedir", + ) # type: ignore[assignment] + + no_verify: bool = Field( + default=False, + description=textwrap.dedent( + """ + Disable signature verification of packaged used for build. + """ + ), + ) # type: ignore[assignment] + + builtin_signature_check: bool = Field( + default=True, + description=textwrap.dedent( + """ + Use the RPM's built-in package signature verification. + """ + ), + ) # type: ignore[assignment] + + disable_hdrmd5_check: bool = Field( + default=False, + description=textwrap.dedent( + """ + Disable hdrmd5 checks of downloaded and cached packages in ``osc build``. + It is recommended to keep the check enabled. + + OBS builds the noarch packages once per binary arch. + Such noarch packages are supposed to be nearly identical across all build arches, + any discrepancy in the payload and dependencies is considered a packaging bug. + But to guarantee that the local builds work identically to builds in OBS, + using the arch-specific copy of the noarch package is required. + Unfortunatelly only one of the noarch packages gets distributed + and can be downloaded from a local mirror. + All other noarch packages are available through the OBS API only. + Since there is currently no information about hdrmd5 checksums of published noarch packages, + we download them, verify hdrmd5 and re-download the package from OBS API on mismatch. + + The same can also happen for architecture depend packages when someone is messing around + with the source history or the release number handling in a way that it is not increasing. + + If you want to save some bandwidth and don't care about the exact rebuilds + you can turn this option on to disable hdrmd5 checks completely. + """ + ), + ) # type: ignore[assignment] + + # REQUEST OPTIONS + section_request: str = Field( + default="Request options", + exclude=True, + section=True, + ) # type: ignore[assignment] + + include_request_from_project: bool = Field( + default=True, + description=textwrap.dedent( + """ + When querying requests, show also those that originate in the specified projects. + """ + ), + ) # type: ignore[assignment] + + request_list_days: int = Field( + default=0, + description=textwrap.dedent( + """ + Limit the age of requests shown with ``osc req list`` to the given number of days. + + This is only the default that can be overridden with ``osc request list -D ``. + Use ``0`` for unlimited. + """ + ), + ) # type: ignore[assignment] + + check_for_request_on_action: bool = Field( + default=True, + description=textwrap.dedent( + """ + Check for pending requests after executing an action (e.g. checkout, update, commit). + """ + ), + ) # type: ignore[assignment] + + request_show_interactive: bool = Field( + default=False, + description=textwrap.dedent( + """ + Show requests in the interactive mode by default. + """ + ), + ) # type: ignore[assignment] + + print_web_links: bool = Field( + default=False, + description=textwrap.dedent( + """ + Print links to Web UI that can be directly pasted to a web browser where possible. + """ + ), + ) # type: ignore[assignment] + + request_show_source_buildstatus: bool = Field( + default=False, + description=textwrap.dedent( + """ + """ + ), + ) # type: ignore[assignment] + + submitrequest_accepted_template: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + Template message for accepting a request. + + Supported substitutions: %(reqid)s, %(type)s, %(who)s, %(src_project)s, %(src_package)s, %(src_rev)s, %(tgt_project)s, %(tgt_package)s + + Example: + Hi %(who)s, your request %(reqid)s (type: %(type)s) for %(tgt_project)s/%(tgt_package)s has been accepted. Thank you for your contribuion. + """ + ), + ) # type: ignore[assignment] + + submitrequest_declined_template: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + Template message for declining a request. + + Supported substitutions: %(reqid)s, %(type)s, %(who)s, %(src_project)s, %(src_package)s, %(src_rev)s, %(tgt_project)s, %(tgt_package)s + + Example: + Hi %(who)s, your request %(reqid)s (type: %(type)s) for %(tgt_project)s/%(tgt_package)s has been declined because ... + """ + ), + ) # type: ignore[assignment] + + request_show_review: bool = Field( + default=False, + description=textwrap.dedent( + """ + Review requests interactively. + """ + ), + ) # type: ignore[assignment] + + review_inherit_group: bool = Field( + default=False, + description=textwrap.dedent( + """ + If a review was accepted in interactive mode and a group was specified, + the review will be accepted for this group. + """ + ), + ) # type: ignore[assignment] + + submitrequest_on_accept_action: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + What to do with the source package if the request has been accepted. + If nothing is specified the API default is used. + + Choices: cleanup, update, noupdate + """ + ), + ) # type: ignore[assignment] + + # OBS ATTRIBUTES + section_obs_attributes: str = Field( + default="OBS attributes", + exclude=True, + section=True, + ) # type: ignore[assignment] + + maintained_attribute: str = Field( + default="OBS:Maintained", + ) # type: ignore[assignment] + + maintenance_attribute: str = Field( + default="OBS:MaintenanceProject", + ) # type: ignore[assignment] + + maintained_update_project_attribute: str = Field( + default="OBS:UpdateProject", + ) # type: ignore[assignment] + + # BUILD TOOL OPTIONS + section_build_tool: str = Field( + default="Build tool options", + exclude=True, + section=True, + ) # type: ignore[assignment] + + build_jobs: Optional[int] = Field( + default_factory=os.cpu_count, + description=textwrap.dedent( + """ + The number of parallel processes during the build. + Defaults to the number of available CPU threads. + + Passed as ``--jobs`` to the build tool. + """ + ), + ini_key="build-jobs", + ) # type: ignore[assignment] + + build_type: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + Type of the build environment passed the build tool as the ``--vm-type`` option: + + - : chroot build + - kvm: KVM VM build (needs build-device, build-swap, build-memory) + - xen: XEN VM build (needs build-device, build-swap, build-memory) + - qemu: [EXPERIMENTAL] QEMU VM build + - lxc: [EXPERIMENTAL] LXC build + """ + ), + ini_key="build-type", + ) # type: ignore[assignment] + + build_memory: Optional[int] = Field( + default=None, + description=textwrap.dedent( + """ + The amount of RAM (in MiB) assigned to a build VM. + """ + ), + ini_key="build-memory", + ) # type: ignore[assignment] + + build_root: str = Field( + default="/var/tmp/build-root/%(repo)s-%(arch)s", + description=textwrap.dedent( + """ + Path to the build root directory. + + Supported substitutions: %(repo)s, %(arch)s, %(project)s, %(package)s and %(apihost)s + where ``apihost`` is the hostname extracted from the currently used ``apiurl``. + + Passed as ``--root `` to the build tool. + """ + ), + ini_key="build-root", + ) # type: ignore[assignment] + + build_shell_after_fail: bool = Field( + default=False, + description=textwrap.dedent( + """ + Start a shell prompt in the build environment if a build fails. + + Passed as ``--shell-after-fail`` to the build tool. + """ + ), + ini_key="build-shell-after-fail", + ) # type: ignore[assignment] + + build_uid: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + Numeric uid:gid to use for the abuild user. + Neither of the values should be 0. + This is useful if you are hacking in the buildroot. + This must be set to the same value if the buildroot is re-used. + + Passed as ``--uid `` to the build tool. + """ + ), + ini_key="build-uid", + ) # type: ignore[assignment] + + build_vm_kernel: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + The kernel used in a VM build. + """ + ), + ini_key="build-kernel", + ) # type: ignore[assignment] + + build_vm_initrd: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + The initrd used in a VM build. + """ + ), + ini_key="build-initrd", + ) # type: ignore[assignment] + + build_vm_disk: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + The disk image used as rootfs in a VM build. + + Passed as ``--vm-disk `` to the build tool. + """ + ), + ini_key="build-device", + ) # type: ignore[assignment] + + build_vm_disk_filesystem: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + The file system type of the disk image used as rootfs in a VM build. + Supported values: ext3 (default), ext4, xfs, reiserfs, btrfs. + + Passed as ``--vm-disk-filesystem `` to the build tool. + """ + ), + ini_key="build-vmdisk-filesystem", + ) # type: ignore[assignment] + + build_vm_disk_size: Optional[int] = Field( + default=None, + description=textwrap.dedent( + """ + The size of the disk image (in MiB) used as rootfs in a VM build. + + Passed as ``--vm-disk-size`` to the build tool. + """ + ), + ini_key="build-vmdisk-rootsize", + ) # type: ignore[assignment] + + build_vm_swap: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + Path to the disk image used as a swap for VM builds. + + Passed as ``--swap`` to the build tool. + """ + ), + ini_key="build-swap", + ) # type: ignore[assignment] + + build_vm_swap_size: Optional[int] = Field( + default=None, + description=textwrap.dedent( + """ + The size of the disk image (in MiB) used as swap in a VM build. + + Passed as ``--vm-swap-size`` to the build tool. + """ + ), + ini_key="build-vmdisk-swapsize", + ) # type: ignore[assignment] + + build_vm_user: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + The username of a user used to run QEMU/KVM process. + """ + ), + ini_key="build-vm-user", + ) # type: ignore[assignment] + + icecream: int = Field( + default=0, + description=textwrap.dedent( + """ + Use Icecream distributed compiler. + The value represents the number of parallel build jobs. + + Passed as ``--icecream `` to the build tool. + """ + ), + ) # type: ignore[assignment] + + ccache: bool = Field( + default=False, + description=textwrap.dedent( + """ + Enable compiler cache (ccache) in build roots. + + Passed as ``--ccache`` to the build tool. + """ + ), + ) # type: ignore[assignment] + + sccache: bool = Field( + default=False, + description=textwrap.dedent( + """ + Enable shared compilation cache (sccache) in build roots. Conflicts with ``ccache``. + + Passed as ``--sccache`` to the build tool. + """ + ), + ) # type: ignore[assignment] + + sccache_uri: Optional[str] = Field( + default=None, + description=textwrap.dedent( + """ + Optional URI for sccache storage. + + Supported URIs depend on the sccache configuration. + The URI allows the following substitutions: + + - {pkgname}: name of the package to be build + + Examples: + + - file:///var/tmp/osbuild-sccache-{pkgname}.tar.lzop + - file:///var/tmp/osbuild-sccache-{pkgname}.tar + - redis://127.0.0.1:6379 + + Passed as ``--sccache-uri `` to the build tool. + """ + ), + ) # type: ignore[assignment] + + no_preinstallimage: bool = Field( + default=False, + description=textwrap.dedent( + """ + Do not use preinstall images to initialize build roots. + """ + ), + ) # type: ignore[assignment] + + extra_pkgs: List[str] = Field( + default=[], + description=textwrap.dedent( + """ + Extra packages to install into the build root when building packages locally with ``osc build``. + + This corresponds to ``osc build -x pkg1 -x pkg2 ...``. + The configured values can be overriden from the command-line with ``-x ''``. + + This global setting may leads to dependency problems when the base distro is not providing the package. + Therefore using server-side ``cli_debug_packages`` option instead is recommended. + + Passed as ``--extra-packs `` to the build tool. + """ + ), + ini_key="extra-pkgs", + ini_type="space-separated-list", + ) # type: ignore[assignment] + + # PATHS TO PROGRAMS + section_programs: str = Field( + default="Paths to programs", + exclude=True, + section=True, + ) # type: ignore[assignment] + + build_cmd: str = Field( + default= + shutil.which("build", path="/usr/bin:/usr/lib/build:/usr/lib/obs-build") + or shutil.which("obs-build", path="/usr/bin:/usr/lib/build:/usr/lib/obs-build") + or "/usr/bin/build", + description=textwrap.dedent( + """ + Path to the 'build' tool. + """ + ), + ini_key="build-cmd", + ) # type: ignore[assignment] + + download_assets_cmd: str = Field( + default= + shutil.which("download_assets", path="/usr/lib/build:/usr/lib/obs-build") + or "/usr/lib/build/download_assets", + description=textwrap.dedent( + """ + Path to the 'download_assets' tool used for downloading assets in SCM/Git based builds. + """ + ), + ini_key="download-assets-cmd", + ) # type: ignore[assignment] + + vc_cmd: str = Field( + default=shutil.which("vc", path="/usr/lib/build:/usr/lib/obs-build") or "/usr/lib/build/vc", + description=textwrap.dedent( + """ + Path to the 'vc' tool. + """ + ), + ini_key="vc-cmd", + ) # type: ignore[assignment] + + su_wrapper: str = Field( + default="sudo", + description=textwrap.dedent( + """ + The wrapper to call build tool as root (sudo, su -, ...). + If empty, the build tool runs under the current user wich works only with KVM at this moment. + """ + ), + ini_key="su-wrapper", + ) # type: ignore[assignment] + + +# Generate rst from a model. Use it to generate man page in sphinx. +# This IS NOT a public API function. +def _model_to_rst(cls, title=None, description=None, sections=None, output_file=None): + """ + Generate + """ + def header(text, char="-"): + result = f"{text}\n" + result += f"{'':{char}^{len(text)}}" + return result -new_conf_template = """ -[general] + def bold(text): + return f"**{text}**" + + def italic(text): + return f"*{text}*" -# URL to access API server, e.g. %(apiurl)s -# you also need a section [%(apiurl)s] with the credentials -apiurl = %(apiurl)s - -# Downloaded packages are cached here. Must be writable by you. -#packagecachedir = %(packagecachedir)s - -# Wrapper to call build as root (sudo, su -, ...) -#su-wrapper = %(su-wrapper)s -# set it empty to run build script as user (works only with KVM atm): -#su-wrapper = - -# rootdir to setup the chroot environment -# can contain %%(repo)s, %%(arch)s, %%(project)s, %%(package)s and %%(apihost)s (apihost is the hostname -# extracted from currently used apiurl) for replacement, e.g. -# /srv/oscbuild/%%(repo)s-%%(arch)s or -# /srv/oscbuild/%%(repo)s-%%(arch)s-%%(project)s-%%(package)s -#build-root = %(build-root)s - -# compile with N jobs (default: "getconf _NPROCESSORS_ONLN") -#build-jobs = N - -# build-type to use - values can be (depending on the capabilities of the 'build' script) -# empty - chroot build -# kvm - kvm VM build (needs build-device, build-swap, build-memory) -# xen - xen VM build (needs build-device, build-swap, build-memory) -# experimental: -# qemu - qemu VM build -# lxc - lxc build -#build-type = - -# Execute always a shell prompt on build failure inside of the build environment -#build-shell-after-fail = 1 + def get_type(name, field): + ini_type = field.extra.get("ini_type", None) + if ini_type: + return ini_type + return field.origin_type.__name__ -# build-device is the disk-image file to use as root for VM builds -# e.g. /var/tmp/FILE.root -#build-device = /var/tmp/FILE.root + def get_default(name, field): + if field.default is None: + return None -# build-swap is the disk-image to use as swap for VM builds -# e.g. /var/tmp/FILE.swap -#build-swap = /var/tmp/FILE.swap + ini_type = field.extra.get("ini_type", None) + if ini_type: + return None -# build-kernel is the boot kernel used for VM builds -#build-kernel = /boot/vmlinuz + origin_type = field.origin_type -# build-initrd is the boot initrd used for VM builds -#build-initrd = /boot/initrd + if origin_type == bool: + return str(int(field.default)) -# build-memory is the amount of memory used in the VM -# value in MB - e.g. 512 -#build-memory = 512 + if origin_type in (int, list): + return str(field.default) -# build-vmdisk-rootsize is the size of the disk-image used as root in a VM build -# values in MB - e.g. 4096 -#build-vmdisk-rootsize = 4096 + if origin_type == str: + return f'"{field.default}"' -# build-vmdisk-swapsize is the size of the disk-image used as swap in a VM build -# values in MB - e.g. 1024 -#build-vmdisk-swapsize = 1024 + # TODO: + raise Exception(f"{name} {field}, {origin_type}") -# build-vmdisk-filesystem is the file system type of the disk-image used in a VM build -# values are ext3(default) ext4 xfs reiserfs btrfs -#build-vmdisk-filesystem = ext4 + result = [] -# Numeric uid:gid to assign to the "abuild" user in the build-root -# or "caller" to use the current users uid:gid -# This is convenient when sharing the buildroot with ordinary userids -# on the host. -# This should not be 0 -# build-uid = + if title: + result.append(header(title, char="=")) + result.append("") -# strip leading build time information from the build log -# buildlog_strip_time = 1 + if description: + result.append(description) + result.append("") -# Enable ccache in build roots. -# ccache = 1 + for name, field in cls.__fields__.items(): + extra = field.extra -# Enable sccache in build roots. Conflicts with ccache. -# Equivalent to sccache_uri = file:///var/tmp/osbuild-sccache-{pkgname}.tar -# sccache = 1 + is_section_header = extra.get("section", False) + if is_section_header: + result.append(header(field.default)) + result.append("") + continue -# Optional URI for sccache storage. Maybe a file://, redis:// or other URI supported -# by the configured sccache install. This uri MAY take {pkgname} as a special parameter -# which will be replaced with the name of the package to be built. -# sccache_uri = file:///var/tmp/osbuild-sccache-{pkgname}.tar.lzop -# sccache_uri = file:///var/tmp/osbuild-sccache-{pkgname}.tar -# sccache_uri = redis://127.0.0.1:6379 + skip = extra.get("ini_skip", False) or field.exclude + if skip: + continue -# extra packages to install when building packages locally (osc build) -# this corresponds to osc build's -x option and can be overridden with that -# -x '' can also be given on the command line to override this setting, or -# you can have an empty setting here. This global setting may leads to -# dependency problems when the base distro is not providing the package. -# => using server side definition via cli_debug_packages substitute rule is -# recommended therefore. -#extra-pkgs = + ini_key = extra.get("ini_key", name) -# build platform is used if the platform argument is omitted to osc build -#build_repository = %(build_repository)s + x = bold(ini_key) + " : " + get_type(name, field) + default = get_default(name, field) + if default: + x += " = " + italic(default) + result.append(x) + result.append("") + desc = field.__doc__ or "" + for line in desc.splitlines(): + result.append(f" {line}") + result.append("") -# default project for getpac or bco -#getpac_default_project = %(getpac_default_project)s + sections = sections or {} + for section_name, section_class in sections.items(): + result.append(header(section_name)) + result.append(_model_to_rst(section_class)) -# alternate filesystem layout: have multiple subdirs, where colons were. -#checkout_no_colon = %(checkout_no_colon)s + if output_file: + with open(output_file, "w", encoding="utf-8") as f: + f.write("\n".join(result)) -# instead of colons, use the specified as separator -#project_separator = %(project_separator)s + return "\n".join(result) -# change filesystem layout: avoid checkout within a project or package dir. -#checkout_rooted = %(checkout_rooted)s -# local files to ignore with status, addremove, .... -#exclude_glob = %(exclude_glob)s +# being global to this module, this object can be accessed from outside +# it will hold the parsed configuration +config = Options() -# limit the age of requests shown with 'osc req list'. -# this is a default only, can be overridden by 'osc req list -D NNN' -# Use 0 for unlimted. -#request_list_days = %(request_list_days)s -# show info useful for debugging -#debug = 1 +options = Options() +general_opts = [field.extra.get("ini_key", field.name) for field in options.__fields__.values() if not field.exclude] +del options -# show HTTP traffic useful for debugging -#http_debug = 1 +host_options = HostOptions(apiurl="https://example.com", username="test") +api_host_options = [field.extra.get("ini_key", field.name) for field in host_options.__fields__.values() if not field.exclude] +del host_options -# number of retries on HTTP transfer -#http_retries = 3 -# Skip signature verification of packages used for build. -#no_verify = 1 +# HACK: Proxy object that modifies field defaults in the Options class; needed for compatibility with the old DEFAULTS dict; prevents breaking osc-plugin-collab +class Defaults: + def _get_field(self, name): + if hasattr(Options, name): + return getattr(Options, name) -# jump into the debugger in case of errors -#post_mortem = 1 + for i in dir(Options): + field = getattr(Options, i) + if field.extra.get("ini_key", None) == name: + return field -# print call traces in case of errors -#traceback = 1 + def __getitem__(self, name): + field = self._get_field(name) + result = field.default + if field.type is List[str]: + # return list as a string so we can append another string to it + return ", ".join(result) + return result -# check for unversioned/removed files before commit -#check_filelist = 1 + def __setitem__(self, name, value): + obj = Options() + obj.set_value_from_string(name, value) + field = self._get_field(name) + field.default = obj[name] -# check for pending requests after executing an action (e.g. checkout, update, commit) -#check_for_request_on_action = 1 -# what to do with the source package if the submitrequest has been accepted. If -# nothing is specified the API default is used -#submitrequest_on_accept_action = cleanup|update|noupdate +DEFAULTS = Defaults() -# template for an accepted submitrequest -#submitrequest_accepted_template = Hi %%(who)s,\\n -# thanks for working on:\\t%%(tgt_project)s/%%(tgt_package)s. -# SR %%(reqid)s has been accepted.\\n\\nYour maintainers -# template for a declined submitrequest -#submitrequest_declined_template = Hi %%(who)s,\\n -# sorry your SR %%(reqid)s (request type: %%(type)s) for -# %%(tgt_project)s/%%(tgt_package)s has been declined because... +new_conf_template = """ +# see oscrc(5) man page for the full list of available options -#review requests interactively (default: off) -#request_show_review = 1 +[general] -# if a review is accepted in interactive mode and a group -# was specified the review will be accepted for this group (default: off) -#review_inherit_group = 1 +# Default URL to the API server. +# Credentials and other `apiurl` specific settings must be configured in a `[$apiurl]` config section. +apiurl=%(apiurl)s [%(apiurl)s] -# set aliases for this apiurl -# aliases = foo, bar -# real name used in .changes, unless the one from osc meta prj will be used -# realname = -# email used in .changes, unless the one from osc meta prj will be used -# email = -# additional headers to pass to a request, e.g. for special authentication -#http_headers = Host: foofoobar, -# User: mumblegack -# Plain text password -#pass = +# aliases= +# user= +# pass= +# credentials_mgr_class=osc.credentials... """ @@ -460,6 +1284,10 @@ def apply_option_types(config, conffile=""): """ +def sanitize_apiurl(apiurl): + return urljoin(*parse_apisrv_url(None, apiurl)) + + def parse_apisrv_url(scheme, apisrv): if apisrv.startswith('http://') or apisrv.startswith('https://'): url = apisrv @@ -477,7 +1305,7 @@ def urljoin(scheme, apisrv, path=''): def is_known_apiurl(url): """returns ``True`` if url is a known apiurl""" - apiurl = urljoin(*parse_apisrv_url(None, url)) + apiurl = sanitize_apiurl(url) return apiurl in config['api_host_options'] @@ -505,7 +1333,7 @@ def get_apiurl_api_host_options(apiurl): # knows this instead of having to extract it from a url where it # had been mingled into before. But this works fine for now. - apiurl = urljoin(*parse_apisrv_url(None, apiurl)) + apiurl = sanitize_apiurl(apiurl) if is_known_apiurl(apiurl): return config['api_host_options'][apiurl] raise oscerr.ConfigMissingApiurl('missing credentials for apiurl: \'%s\'' % apiurl, @@ -543,7 +1371,7 @@ def get_configParser(conffile=None, force_read=False): if 'conffile' not in get_configParser.__dict__: get_configParser.conffile = conffile if force_read or 'cp' not in get_configParser.__dict__ or conffile != get_configParser.conffile: - get_configParser.cp = OscConfigParser.OscConfigParser(DEFAULTS) + get_configParser.cp = OscConfigParser.OscConfigParser() get_configParser.cp.read(conffile) get_configParser.conffile = conffile return get_configParser.cp @@ -590,8 +1418,7 @@ def config_set_option(section, opt, val=None, delete=False, update=True, creds_m config/reset to the default value. """ cp = get_configParser(config['conffile']) - # don't allow "internal" options - general_opts = [i for i in DEFAULTS.keys() if i not in ['user', 'pass', 'passx']] + if section != 'general': section = config['apiurl_aliases'].get(section, section) scheme, host, path = \ @@ -686,10 +1513,10 @@ def write_initial_config(conffile, entries, custom_template='', creds_mgr_descri custom_template is an optional configuration template. """ conf_template = custom_template or new_conf_template - config = DEFAULTS.copy() + config = globals()["config"].dict() config.update(entries) sio = StringIO(conf_template.strip() % config) - cp = OscConfigParser.OscConfigParser(DEFAULTS) + cp = OscConfigParser.OscConfigParser() cp.readfp(sio) cp.set(config['apiurl'], 'user', config['user']) if creds_mgr_descriptor: @@ -732,8 +1559,6 @@ def _get_credentials_manager(url, cp): return creds_mgr if config['use_keyring'] and GENERIC_KEYRING: return credentials.get_keyring_credentials_manager(cp) - elif cp.get(url, 'passx') is not None: - return credentials.ObfuscatedConfigFileCredentialsManager(cp, None) return credentials.PlaintextConfigFileCredentialsManager(cp, None) @@ -758,22 +1583,46 @@ def get_config(override_conffile=None, override_verbose=None, overrides=None ): - """do the actual work (see module documentation)""" - global config - - if not override_conffile: - conffile = identify_conf() + if overrides: + overrides = overrides.copy() else: + overrides = {} + + if override_apiurl is not None: + overrides["apiurl"] = override_apiurl + + if override_debug is not None: + overrides["debug"] = override_debug + + if override_http_debug is not None: + overrides["http_debug"] = override_http_debug + + if override_http_full_debug is not None: + overrides["http_debug"] = override_http_full_debug or overrides["http_debug"] + overrides["http_full_debug"] = override_http_full_debug + + if override_traceback is not None: + overrides["traceback"] = override_traceback + + if override_post_mortem is not None: + overrides["post_mortem"] = override_post_mortem + + if override_no_keyring is not None: + overrides["use_keyring"] = not override_no_keyring + + if override_verbose is not None: + overrides["verbose"] = override_verbose + + if override_conffile is not None: conffile = override_conffile + else: + conffile = identify_conf() conffile = os.path.expanduser(conffile) if not os.path.exists(conffile): - raise oscerr.NoConfigfile(conffile, - account_not_configured_text % conffile) - - # okay, we made sure that oscrc exists + raise oscerr.NoConfigfile(conffile, account_not_configured_text % conffile) - # make sure it is not world readable, it may contain a password. + # make sure oscrc is not world readable, it may contain a password conffile_stat = os.stat(conffile) if conffile_stat.st_mode != 0o600: try: @@ -789,170 +1638,81 @@ def get_config(override_conffile=None, if not cp.has_section('general'): # FIXME: it might be sufficient to just assume defaults? msg = config_incomplete_text % conffile - msg += new_conf_template % DEFAULTS + defaults = Options().dict() + msg += new_conf_template % defaults raise oscerr.ConfigError(msg, conffile) - config = dict(cp.items('general', raw=1)) - - # if the overrides trigger an exception, the 'post_mortem' option - # must be set to the appropriate type otherwise the non-empty string gets evaluated as True - config = apply_option_types(config, conffile) - - overrides = overrides or {} - for key, value in overrides.items(): - if key not in config: - raise oscerr.ConfigError(f"Unknown config option '{key}'", "") - config[key] = value - - config['conffile'] = conffile - - config = apply_option_types(config, conffile) - - config['packagecachedir'] = os.path.expanduser(config['packagecachedir']) - config['exclude_glob'] = config['exclude_glob'].split() + global config - re_clist = re.compile('[, ]+') - config['extra-pkgs'] = [i.strip() for i in re_clist.split(config['extra-pkgs'].strip()) if i] - config["exclude_files"] = [i.strip() for i in re_clist.split(config["exclude_files"].strip()) if i] - config["include_files"] = [i.strip() for i in re_clist.split(config["include_files"].strip()) if i] + config = Options() + config.conffile = conffile - # collect the usernames, passwords and additional options for each api host - api_host_options = {} + # read host options first in order to populate apiurl aliases + urls = [i for i in cp.sections() if i != "general"] + for url in urls: + apiurl = sanitize_apiurl(url) + username = cp[url]["user"] - # Regexp to split extra http headers into a dictionary - # the text to be matched looks essentially looks this: - # "Attribute1: value1, Attribute2: value2, ..." - # there may be arbitray leading and intermitting whitespace. - # the following regexp does _not_ support quoted commas within the value. - http_header_regexp = re.compile(r"\s*(.*?)\s*:\s*(.*?)\s*(?:,\s*|\Z)") + host_options = HostOptions(apiurl=apiurl, username=username) + # TODO: inherit from main config? a Field keyword inherit=(object, attr_name?) - # override values which we were called with - # This needs to be done before processing API sections as it might be already used there - if override_no_keyring: - config['use_keyring'] = False + for name, field in host_options.__fields__.items(): + ini_key = field.extra.get("ini_key", name) - aliases = {} - for url in [x for x in cp.sections() if x != 'general']: - # backward compatiblity - scheme, host, path = parse_apisrv_url(config.get('scheme', 'https'), url) - apiurl = urljoin(scheme, host, path) - creds_mgr = _get_credentials_manager(url, cp) - # if the deprecated gnomekeyring is used we should use the apiurl instead of url - # (that's what the old code did), but this makes things more complex - # (also, it is very unlikely that url and apiurl differ) - user = _extract_user_compat(cp, url, creds_mgr) - if user is None: - raise oscerr.ConfigMissingCredentialsError('No user found in section %s' % url, conffile, url) - password = creds_mgr.get_password(url, user, defer=True, apiurl=apiurl) - if password is None: - raise oscerr.ConfigMissingCredentialsError('No password found in section %s' % url, conffile, url) - - if cp.has_option(url, 'http_headers'): - http_headers = cp.get(url, 'http_headers') - http_headers = http_header_regexp.findall(http_headers) - else: - http_headers = [] - if cp.has_option(url, 'aliases'): - for i in cp.get(url, 'aliases').split(','): - key = i.strip() - if key == '': - continue - if key in aliases: - msg = 'duplicate alias entry: \'%s\' is already used for another apiurl' % key - raise oscerr.ConfigError(msg, conffile) - aliases[key] = url - - entry = {'user': user, - 'pass': password, - 'http_headers': http_headers} - api_host_options[apiurl] = APIHostOptionsEntry(entry) - - optional = ( - 'realname', 'email', 'sslcertck', 'cafile', 'capath', 'sshkey', 'allow_http', - credentials.AbstractCredentialsManager.config_entry, - ) - for key in optional: - if not cp.has_option(url, key): - continue - if key in _boolean_opts: - api_host_options[apiurl][key] = cp.getboolean(url, key) - elif key in _integer_opts: - api_host_options[apiurl][key] = cp.getint(url, key) + if ini_key in cp[url]: + value = cp[url][ini_key] else: - api_host_options[apiurl][key] = cp.get(url, key) + continue - if cp.has_option(url, 'build-root', proper=True): - api_host_options[apiurl]['build-root'] = cp.get(url, 'build-root', raw=True) + if name == "password": + creds_mgr = _get_credentials_manager(url, cp) + value = creds_mgr.get_password(url, host_options.username, defer=True, apiurl=host_options.apiurl) + if value is None: + raise oscerr.ConfigMissingCredentialsError("No password found in section {url}", conffile, url) + + host_options.set_value_from_string(name, value) + + scheme = urlsplit(apiurl)[0] + if scheme == "http" and not host_options.allow_http: + msg = "The apiurl '{apiurl}' uses HTTP protocol without any encryption.\n" + msg += "All communication incl. sending your password IS NOT ENCRYPTED!\n" + msg += "Add 'allow_http=1' to the [{apiurl}] config file section to mute this message.\n" + print(msg.format(apiurl=apiurl), file=sys.stderr) + + config.api_host_options[apiurl] = host_options + + # read the main options + for name, field in config.__fields__.items(): + ini_key = field.extra.get("ini_key", name) + env_key = f"OSC_{name.upper()}" + + # priority: env, overrides, config + if env_key in os.environ: + value = os.environ["env_key"] + elif name in overrides: + value = overrides.pop(name) + elif ini_key in overrides: + value = overrides.pop(ini_key) + elif ini_key in cp["general"]: + value = cp["general"][ini_key] + else: + continue - if 'sslcertck' not in api_host_options[apiurl]: - api_host_options[apiurl]['sslcertck'] = True + if name == "apiurl": + # resolve an apiurl alias to an actual apiurl + apiurl = config.apiurl_aliases.get(value, None) + if not apiurl: + # no alias matched, try again with a sanitized apiurl (with https:// prefix) + # and if there's no match again, just use the sanitized apiurl + apiurl = sanitize_apiurl(value) + apiurl = config.apiurl_aliases.get(apiurl, apiurl) + value = apiurl - if 'allow_http' not in api_host_options[apiurl]: - api_host_options[apiurl]['allow_http'] = False + config.set_value_from_string(name, value) - if cp.has_option(url, 'trusted_prj'): - api_host_options[apiurl]['trusted_prj'] = cp.get(url, 'trusted_prj').split(' ') - else: - api_host_options[apiurl]['trusted_prj'] = [] - - # This option is experimental and may be removed at any time in the future! - # This allows overriding the download url for an OBS instance to specify a closer mirror - # or proxy system, which can greatly improve download performance, latency and more. - # For example, this can use https://github.com/Firstyear/opensuse-proxy-cache in a local - # geo to improve performance. - if cp.has_option(url, 'downloadurl'): - api_host_options[apiurl]['downloadurl'] = cp.get(url, 'downloadurl') - else: - api_host_options[apiurl]['downloadurl'] = None - - if api_host_options[apiurl]['sshkey'] is None: - api_host_options[apiurl]['sshkey'] = config['sshkey'] - - api_host_options[apiurl]["disable_hdrmd5_check"] = config["disable_hdrmd5_check"] - if cp.has_option(url, "disable_hdrmd5_check"): - api_host_options[apiurl]["disable_hdrmd5_check"] = cp.getboolean(url, "disable_hdrmd5_check") - - # add the auth data we collected to the config dict - config['api_host_options'] = api_host_options - config['apiurl_aliases'] = aliases - - apiurl = aliases.get(config['apiurl'], config['apiurl']) - config['apiurl'] = urljoin(*parse_apisrv_url(None, apiurl)) - # backward compatibility - if 'apisrv' in config: - apisrv = config['apisrv'].lstrip('http://') - apisrv = apisrv.lstrip('https://') - scheme = config.get('scheme', 'https') - config['apiurl'] = urljoin(scheme, apisrv) - if 'apisrc' in config or 'scheme' in config: - print('Warning: Use of the \'scheme\' or \'apisrv\' in oscrc is deprecated!\n' - 'Warning: See README for migration details.', file=sys.stderr) - if 'build_platform' in config: - print('Warning: Use of \'build_platform\' config option is deprecated! (use \'build_repository\' instead)', file=sys.stderr) - config['build_repository'] = config['build_platform'] - - config['verbose'] = bool(int(config['verbose'])) - # override values which we were called with - if override_verbose is not None: - config['verbose'] = bool(override_verbose) - - config['debug'] = bool(int(config['debug'])) - if override_debug is not None: - config['debug'] = bool(override_debug) - - if override_http_debug: - config['http_debug'] = override_http_debug - if override_http_full_debug: - config['http_debug'] = override_http_full_debug or config['http_debug'] - config['http_full_debug'] = override_http_full_debug - if override_traceback: - config['traceback'] = override_traceback - if override_post_mortem: - config['post_mortem'] = override_post_mortem - if override_apiurl: - apiurl = aliases.get(override_apiurl, override_apiurl) - # check if apiurl is a valid url - config['apiurl'] = urljoin(*parse_apisrv_url(None, apiurl)) + if overrides: + unused_overrides_str = ", ".join((f"'{i}'" for i in overrides)) + raise oscerr.ConfigError(f"Unknown config options: {unused_overrides_str}", "") # XXX unless config['user'] goes away (and is replaced with a handy function, or # config becomes an object, even better), set the global 'user' here as well, @@ -964,13 +1724,6 @@ def get_config(override_conffile=None, e.file = conffile raise e - scheme = urlsplit(apiurl)[0] - if scheme == "http" and not api_host_options[apiurl]['allow_http']: - msg = "The apiurl '{apiurl}' uses HTTP protocol without any encryption.\n" - msg += "All communication incl. sending your password IS NOT ENCRYPTED!\n" - msg += "Add 'allow_http=1' to the [{apiurl}] config file section to mute this message.\n" - print(msg.format(apiurl=apiurl), file=sys.stderr) - # enable connection debugging after all config options are set from .connection import enable_http_debug enable_http_debug(config) @@ -993,7 +1746,7 @@ def identify_conf(): def interactive_config_setup(conffile, apiurl, initial=True): if not apiurl: - apiurl = DEFAULTS["apiurl"] + apiurl = Options()["apiurl"] scheme = urlsplit(apiurl)[0] http = scheme == "http" diff --git a/osc/credentials.py b/osc/credentials.py index 218d3d04b5..8b77de317c 100644 --- a/osc/credentials.py +++ b/osc/credentials.py @@ -19,41 +19,6 @@ from . import oscerr -class _LazyPassword: - def __init__(self, pwfunc): - self._pwfunc = pwfunc - self._password = None - - def __str__(self): - if self._password is None: - password = self._pwfunc() - if callable(password): - print('Warning: use of a deprecated credentials manager API.', - file=sys.stderr) - password = password() - if password is None: - raise oscerr.OscIOError(None, 'Unable to retrieve password') - self._password = password - return self._password - - def __format__(self, format_spec): - if format_spec.endswith("s"): - return f"{self.__str__():{format_spec}}" - return super().__format__(format_spec) - - def __len__(self): - return len(str(self)) - - def __add__(self, other): - return str(self) + other - - def __radd__(self, other): - return other + str(self) - - def __getattr__(self, name): - return getattr(str(self), name) - - class AbstractCredentialsManagerDescriptor: def name(self): raise NotImplementedError() @@ -90,9 +55,9 @@ def _get_password(self, url, user, apiurl=None): def get_password(self, url, user, defer=True, apiurl=None): if defer: - return _LazyPassword(lambda: self._get_password(url, user, apiurl=apiurl)) + return conf.Password(lambda: self._get_password(url, user, apiurl=apiurl)) else: - return self._get_password(url, user, apiurl=apiurl) + return conf.Password(self._get_password(url, user, apiurl=apiurl)) def set_password(self, url, user, password): raise NotImplementedError() diff --git a/tests/test_conf.py b/tests/test_conf.py index 48d1188085..163cf8286f 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -70,6 +70,8 @@ linkcontrol = 0 include_request_from_project = 1 local_service_run = 1 +include_files = incl *.incl +exclude_files = excl *.excl maintained_attribute = OBS:Maintained maintenance_attribute = OBS:MaintenanceProject maintained_update_project_attribute = OBS:UpdateProject @@ -84,7 +86,8 @@ passx = unused aliases = osc http_headers = - authorization: Basic QWRtaW46b3BlbnN1c2U= + Authorization: Basic QWRtaW46b3BlbnN1c2U= + X-Foo: Bar realname = The Administrator email = admin@example.com sslcertck = 1 @@ -309,6 +312,12 @@ def test_include_request_from_project(self): def test_local_service_run(self): self.assertEqual(self.config["local_service_run"], True) + def test_exclude_files(self): + self.assertEqual(self.config["exclude_files"], ["excl", "*.excl"]) + + def test_include_files(self): + self.assertEqual(self.config["include_files"], ["incl", "*.incl"]) + def test_maintained_attribute(self): self.assertEqual(self.config["maintained_attribute"], "OBS:Maintained") @@ -339,7 +348,10 @@ def test_host_option_http_headers(self): host_options = self.config["api_host_options"][self.config["apiurl"]] self.assertEqual( host_options["http_headers"], - [("authorization", "Basic QWRtaW46b3BlbnN1c2U=")], + [ + ("Authorization", "Basic QWRtaW46b3BlbnN1c2U="), + ("X-Foo", "Bar"), + ], ) def test_host_option_realname(self): diff --git a/tests/test_output.py b/tests/test_output.py index 87fb4dcca3..8c02adbcec 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -82,7 +82,7 @@ def tearDown(self): importlib.reload(osc.conf) def test_debug(self): - osc.conf.config["debug"] = 0 + osc.conf.config["debug"] = False stdout = io.StringIO() stderr = io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): @@ -90,7 +90,7 @@ def test_debug(self): self.assertEqual("", stdout.getvalue()) self.assertEqual("", stderr.getvalue()) - osc.conf.config["debug"] = 1 + osc.conf.config["debug"] = True stdout = io.StringIO() stderr = io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): @@ -99,7 +99,7 @@ def test_debug(self): self.assertEqual("DEBUG: foo bar\n", stderr.getvalue()) def test_verbose(self): - osc.conf.config["verbose"] = 0 + osc.conf.config["verbose"] = False stdout = io.StringIO() stderr = io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): @@ -107,7 +107,7 @@ def test_verbose(self): self.assertEqual("", stdout.getvalue()) self.assertEqual("", stderr.getvalue()) - osc.conf.config["verbose"] = 1 + osc.conf.config["verbose"] = True stdout = io.StringIO() stderr = io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): @@ -115,8 +115,8 @@ def test_verbose(self): self.assertEqual("foo bar\n", stdout.getvalue()) self.assertEqual("", stderr.getvalue()) - osc.conf.config["verbose"] = 0 - osc.conf.config["debug"] = 1 + osc.conf.config["verbose"] = False + osc.conf.config["debug"] = True stdout = io.StringIO() stderr = io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):