From e0b2b9d09a4ef86b7e93736ebb1ca88acf480a1b Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Sat, 16 Dec 2023 13:51:57 -0500 Subject: [PATCH] * remove deprecated code * fix mailmap * add more authors by contribution * remove the ini in cloudmesh so we get by default a namespace --- .mailmap | 26 ++ AUTHORS | 32 +- Makefile | 2 + cloudmesh/common/__init__.py | 1 - cloudmesh/common/bin/readme.py | 2 +- deprecated.travis.yml | 51 --- deprecated/BaseConfigDict.py | 536 ----------------------- deprecated/ConfigDict.py | 522 ---------------------- deprecated/config.py | 31 -- deprecated/deprecated_stopwatch_mllog.py | 447 ------------------- deprecated/etc/__init__.py | 0 deprecated/etc/cloudmesh.yaml | 227 ---------- deprecated/old_init.py | 20 - deprecated/test_configdict.py | 108 ----- makefile-info.mk | 6 + 15 files changed, 61 insertions(+), 1950 deletions(-) delete mode 100644 cloudmesh/common/__init__.py delete mode 100644 deprecated.travis.yml delete mode 100644 deprecated/BaseConfigDict.py delete mode 100644 deprecated/ConfigDict.py delete mode 100644 deprecated/config.py delete mode 100644 deprecated/deprecated_stopwatch_mllog.py delete mode 100644 deprecated/etc/__init__.py delete mode 100644 deprecated/etc/cloudmesh.yaml delete mode 100644 deprecated/old_init.py delete mode 100644 deprecated/test_configdict.py create mode 100644 makefile-info.mk diff --git a/.mailmap b/.mailmap index 0fcc0b60..c039765d 100644 --- a/.mailmap +++ b/.mailmap @@ -1,3 +1,29 @@ +Gregor von Laszewski +J.P Fleischer <70083705+stapmoshun@users.noreply.github.com> +J.P Fleischer +Robert Knuuti +Fugang Wang +Badi Abdul-Wahid +Andrew Holland +J.P Fleisher <70083705+jpfleischer@users.noreply.github.com> +Rick Otten +Robert Knuuti +Dave DeMeulenaere +Lars Olson +Anthon van der Neut +Anthony Orlowski +Jackson Miskill <103867645+j-miskill@users.noreply.github.com> +Toble007 <74217657+Toble007@users.noreply.github.com> +J.P. Fleischer <70083705+jpfleischer@users.noreply.github.com> +Ketan Pimparkar <43309148+kpimparkar@users.noreply.github.com> +Vafa Andalibi +Alex Beck <105373869+abeck14@users.noreply.github.com> +Anthony Orlowski +Anthony Orlowski +Ashok +Karthick +Rick Otten +Dhakshesh Kolli Andrew Holland <54564354+ElectricErudite@users.noreply.github.com> Andrew Holland Anthon van der Neut diff --git a/AUTHORS b/AUTHORS index 1e8e6baa..5394305d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,9 +4,29 @@ Author Contributors - fugangwang - Badi Abdul-Wahid - Anthon van der Neut - Dave DeMeulenaere - justbbusy - karthick + 1768 Gregor von Laszewski + 113 J.P <70083705+stapmoshun@users.noreply.github.com> + 21 J.P + 20 Robert Knuuti + 17 Fugang Wang + 16 Badi Abdul-Wahid + 10 Andrew Holland + 8 J.P <70083705+jpfleischer@users.noreply.github.com> + 8 rickotten + 7 Robert Knuuti + 6 Dave DeMeulenaere + 5 Lars Olson + 4 Anthon van der Neut + 4 Anthony Orlowski + 4 Jackson Miskill <103867645+j-miskill@users.noreply.github.com> + 4 Toble007 <74217657+Toble007@users.noreply.github.com> + 3 jpfleischer <70083705+jpfleischer@users.noreply.github.com> + 2 Ketan Pimparkar <43309148+kpimparkar@users.noreply.github.com> + 2 Vafa Andalibi + 2 abeck14 <105373869+abeck14@users.noreply.github.com> + 2 aporlowski + 1 Anthony Orlowski + 1 Ashok + 1 Karthick + 1 devrick + 1 dkkolli diff --git a/Makefile b/Makefile index 49332162..d7ab8e6d 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ package=common include makefile-basic.mk +include makefile-info.mk + include makefile-test.mk include makefile-clean.mk diff --git a/cloudmesh/common/__init__.py b/cloudmesh/common/__init__.py deleted file mode 100644 index 6a92b244..00000000 --- a/cloudmesh/common/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '5.0.2' diff --git a/cloudmesh/common/bin/readme.py b/cloudmesh/common/bin/readme.py index 4962bd67..6d96f364 100644 --- a/cloudmesh/common/bin/readme.py +++ b/cloudmesh/common/bin/readme.py @@ -13,7 +13,7 @@ command = sys.argv[2] warning = f""" -> **Note:** The README.md page is outomatically generated, do not edit it. +> **Note:** The README.md page is automatically generated, do not edit it. > To modify change the content in > > Curley brackets must use two in README-source.md diff --git a/deprecated.travis.yml b/deprecated.travis.yml deleted file mode 100644 index 24dca684..00000000 --- a/deprecated.travis.yml +++ /dev/null @@ -1,51 +0,0 @@ -os: - - linux -# - windows - -# -# On OSX we get currently the following error, so we outcomment buildo n osx for now -# -# - osx -# -# 3.7 is not installed; attempting download -# Downloading archive: https://storage.googleapis.com/travis-ci-language-archives/python/binaries/osx/10.13/x86_64/python-3.7.tar.bz2 -# 0.14s$ curl -sSf --retry 5 -o python-3.7.tar.bz2 ${archive_url} -# curl: (22) The requested URL returned error: 404 -# Unable to download 3.7 archive. The archive may not exist. Please consider a different version. - - -arch: arm64 - -dist: bionic - -sudo: required - -language: python - -python: - - "3.8.2" - -install: - - pip install pip -U - - python --version - - pip --version - - pwd - - pip install -r requirements.txt -# - pip install -r requirements-dev.txt - - pip install . -# - mkdir ~/.cloudmesh -# - cp cloudmesh/etc/cloudmesh.yaml ~/.cloudmesh - -script: - - pytest -v --capture=no tests/test_stopwatch.py - - pytest -v --capture=no tests/test_strdb.py - - pytest -v --capture=no tests/test_tables.py - - pytest -v --capture=no tests/test_flatdict.py - - pytest -v --capture=no tests/test_printer.py - - pytest -v --capture=no tests/test_shell.py - - pytest -v --capture=no tests/test_parameter.py - - pytest -v --capture=no tests/test_tables.py - - pytest -v --capture=no tests/test_tabulate.py - -# remove the verbose test as it does not work in python 2.7 -# - pytest -v tests/test_verbose.py diff --git a/deprecated/BaseConfigDict.py b/deprecated/BaseConfigDict.py deleted file mode 100644 index f13e1361..00000000 --- a/deprecated/BaseConfigDict.py +++ /dev/null @@ -1,536 +0,0 @@ -"""Some simple yaml file reader""" - -import ast -import json -import os -import stat -import sys -from collections import OrderedDict -from pprint import pprint -from string import Template - -import simplejson -import oyaml as yaml - -from cloudmesh.common.console import Console -from cloudmesh.common.error import Error -from cloudmesh.common.location import config_file -from cloudmesh.common.util import backup_name, path_expand - -import warnings - -# warnings.simplefilter('ignore', yaml.error.UnsafeLoaderWarning) -# Logger dependency not to be there in utility -# log = LOGGER(__file__) -package_dir = os.path.dirname(os.path.abspath(__file__)) -attribute_indent = 4 - - -def check_file_for_tabs(filename, verbose=True): - """identifies if the file contains tabs and returns True if it - does. It also prints the location of the lines and columns. If - verbose is set to False, the location is not printed. - - :param verbose: if true prints information about issues - :param filename: the filename - :rtype: True if there are tabs in the file - """ - file_contains_tabs = False - with open(filename) as f: - lines = f.read().splitlines() - - line_no = 1 - for line in lines: - if "\t" in line: - file_contains_tabs = True - location = [ - i for i in range(len(line)) if line.startswith('\t', i)] - if verbose: - Console.error("Tab found in line {} and column(s) {}" - .format(line_no, - str(location).replace("[", "").replace( - "]", "")), - traceflag=False) - line_no += 1 - return file_contains_tabs - - -# http://stackoverflow.com/questions/5121931/in-python-how-can-you-load-yaml-mappings-as-ordereddicts - -# noinspection PyPep8Naming -def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict): - """ - Loads an ordered dict into a yaml while preserving the order - - :param stream: the name of the stream - :param Loader: the yam loader (such as yaml.SafeLoader) - :param object_pairs_hook: the ordered dict - """ - - # noinspection PyClassHasNoInit - class OrderedLoader(Loader): - """ - A helper class to define an Ordered Loader - """ - pass - - def construct_mapping(loader, node): - """ - construct a flattened node mapping - :param loader: the loader - :param node: the node dict - :return: - """ - loader.flatten_mapping(node) - return object_pairs_hook(loader.construct_pairs(node)) - - OrderedLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - construct_mapping) - return yaml.load(stream, OrderedLoader) - - -# usage example: -# ordered_load(stream, yaml.SafeLoader) - - -# noinspection PyPep8Naming -def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **keywords): - """ - writes the dict into an ordered yaml. - - :param data: The ordered dict - :param stream: the stream - :param Dumper: the dumper such as yaml.SafeDumper - """ - - # noinspection PyClassHasNoInit - class OrderedDumper(Dumper): - """ - A helper class to create an ordered dump - """ - pass - - def _dict_representer(dumper, data): - return dumper.represent_mapping( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - data.items()) - - OrderedDumper.add_representer(OrderedDict, _dict_representer) - return yaml.dump(data, stream, OrderedDumper, **keywords) - - -# usage: -# ordered_dump(data, Dumper=yaml.SafeDumper) - - -def read_yaml_config(filename, check=True, osreplace=True, exit=True): - """ - reads in a yaml file from the specified filename. If check is set to true - the code will fail if the file does not exist. However if it is set to - false and the file does not exist, None is returned. - - :param exit: if true is exist with sys exit - :param osreplace: if true replaces environment variables from the OS - :param filename: the file name - :param check: if True fails if the file does not exist, - if False and the file does not exist return will be None - """ - location = filename - if location is not None: - location = path_expand(location) - - if not os.path.exists(location) and not check: - return None - - if check and os.path.exists(location): - - # test for tab in yaml file - if check_file_for_tabs(location): - log.error("The file {0} contains tabs. yaml " - "Files are not allowed to contain tabs".format(location)) - sys.exit() - result = None - try: - - if osreplace: - result = open(location, 'r').read() - t = Template(result) - result = t.substitute(os.environ) - - # data = yaml.safe_load(result) - data = ordered_load(result, yaml.SafeLoader) - else: - f = open(location, "r") - - # data = yaml.safe_load(f) - - data = ordered_load(result, yaml.SafeLoader) - f.close() - - return data - except Exception as e: - log.error( - "The file {0} fails with a yaml read error".format(filename)) - Error.traceback(e) - sys.exit() - - else: - log.error("The file {0} does not exist.".format(filename)) - if exit: - sys.exit() - - return None - - -class OrderedJsonEncoder(simplejson.JSONEncoder): - """ - Manage ordered Json Objects - """ - indent = attribute_indent - - def encode(self, o, depth=0): - """ - encode the json object at given depth - :param o: the object - :param depth: the depth - :return: the json encoding - """ - if isinstance(o, OrderedDict): - return "{" + ",\n ".join([self.encode(k) + ":" + - self.encode(v, depth + 1) - for (k, v) in o.items()]) + "}\n" - else: - return simplejson.JSONEncoder.encode(self, o) - - -def custom_print(data_structure, indent): - """ - prints a given data structure such as a dict or ordered dict at a given indentation level - :param data_structure: - :param indent: - :return: - """ - for key, value in data_structure.items(): - print("\n%s%s:" % (' ' * attribute_indent * indent, str(key)), end=' ') - if isinstance(value, OrderedDict): - custom_print(value, indent + 1) - elif isinstance(value, dict): - custom_print(value, indent + 1) - else: - print("%s" % (str(value)), end=' ') - - -class BaseConfigDict(OrderedDict): - """ - A class to obtain an OrderedDict from a yaml file. - """ - - def _set_filename(self, filename): - """ - Sets the filename to be used. - - :param filename: the filename - """ - self['filename'] = filename - self['location'] = path_expand(self["filename"]) - - def __init__(self, *args, **kwargs): - """ - The initialization method - """ - OrderedDict.__init__(self, *args, **kwargs) - - if 'filename' in kwargs: - self._set_filename(kwargs['filename']) - else: - log.error("filename not specified") - # sys.exit() - - if os.path.isfile(self['location']): - self.load(self['location']) - - # print ("ATTRIBUTE", attribute) - for attribute in ['prefix']: - if attribute in kwargs: - self[attribute] = kwargs[attribute] - else: - self[attribute] = None - - self._update_meta() - - def _update_meta(self): - """ - internal function to define the metadata regarding filename, location, - and prefix. - """ - for v in ["filename", "location", "prefix"]: - if "meta" not in self: - self["meta"] = {} - self["meta"][v] = self[v] - del self[v] - - def read(self, filename): - """ - Loads the information in the yaml file. It is the same as load and is - used for compatibility reasons. - - :param filename: the name of the yaml file - """ - self.load(filename) - - def load(self, filename): - """ - Loads the yaml file with the given filename. - - :param filename: the name of the yaml file - """ - - self._set_filename(filename) - - if os.path.isfile(self['location']): - # d = OrderedDict(read_yaml_config(self['location'], check=True)) - d = read_yaml_config(self['location'], check=True) - with open(self['location']) as myfile: - document = myfile.read() - x = yaml.load(document, Loader=yaml.FullLoader) - try: - self.update(d) - except: # noqa: E722 - print("ERROR: can not find", self["location"]) - sys.exit() - else: - print( - "Error while reading and updating the configuration file {:}".format( - filename)) - - def make_a_copy(self, location=None): - """ - Creates a backup of the file specified in the location. The backup - filename appends a .bak.NO where number is a number that is not yet - used in the backup directory. - - TODO: This function should be moved to another file maybe XShell - - :param location: the location of the file to be backed up - """ - import shutil - destination = backup_name(location) - shutil.copyfile(location, destination) - - def write(self, filename=None, output="dict", - attribute_indent=attribute_indent): - """ - This method writes the dict into various output formats. This includes a dict, - json, and yaml - - :param filename: the file in which the dict is written - :param output: is a string that is either "dict", "json", "yaml" - :param attribute_indent: character indentation of nested attributes in - """ - if filename is not None: - location = path_expand(filename) - else: - location = self['meta']['location'] - - # with open('data.yml', 'w') as outfile: - # outfile.write( yaml.dump(data, default_flow_style=True) ) - - # Make a backup - self.make_a_copy(location) - - f = os.open(location, os.O_CREAT | os.O_TRUNC | - os.O_WRONLY, stat.S_IRUSR | stat.S_IWUSR) - if output == "json": - os.write(f, self.json()) - elif output in ['yml', 'yaml']: - # d = dict(self) - # os.write(f, yaml.dump(d, default_flow_style=False)) - os.write(f, ordered_dump(OrderedDict(self), - Dumper=yaml.SafeDumper, - default_flow_style=False, - indent=attribute_indent)) - elif output == "print": - os.write(f, str(custom_print(self, attribute_indent))) - else: - os.write(f, self.dump()) - os.close(f) - - def error_keys_not_found(self, keys): - """ - Check if the requested keys are found in the dict. - - :param keys: keys to be looked for - """ - try: - log.error("Filename: {0}".format(self['meta']['location'])) - except: # noqa: E722 - log.error("Filename: {0}".format(self['location'])) - log.error("Key '{0}' does not exist".format('.'.join(keys))) - indent = "" - last_index = len(keys) - 1 - for i, k in enumerate(keys): - if i == last_index: - log.error(indent + k + ": <- this value is missing") - else: - log.error(indent + k + ":") - indent += " " - - def __str__(self): - """ - returns the json output of the dict. - """ - return self.json() - - def json(self): - """ - returns the json output of the dict. - """ - return json.dumps(self, indent=attribute_indent) - - def yaml(self): - """ - returns the yaml output of the dict. - """ - return ordered_dump(OrderedDict(self), - Dumper=yaml.SafeDumper, - default_flow_style=False) - - # noinspection PyPep8Naming - def dump(self): - """ - returns the json output of the dict. - """ - orderedPrinter = OrderedJsonEncoder() - return orderedPrinter.encode(self) - - def pprint(self): - """ - uses pprint to print the dict - """ - print(custom_print(self, attribute_indent)) - - """ - def __getitem__(self, *mykeys): - try: - item = self.get(mykeys[0]) - except: - self._notify_of_error(mykeys) - sys.exit() - return item - """ - - def get(self, *keys): - """ - returns the dict of the information as read from the yaml file. To - access the file safely, you can use the keys in the order of the - access. - Example: get("provisioner","policy") will return the value of - config["provisioner"]["policy"] from the yaml file if it does not exists - an error will be printing that the value does not exists. Alternatively - you can use the . notation e.g. get("provisioner.policy") - """ - if keys is None: - return self - - if "." in keys[0]: - keys = keys[0].split('.') - element = self - for v in keys: - try: - element = element[v] - except KeyError: - self.error_keys_not_found(keys) - # sys.exit() - return element - - def set(self, value, *keys): - """ - Sets the dict of the information as read from the yaml file. To access - the file safely, you can use the keys in the order of the access. - Example: set("{'project':{'fg82':[i0-i10]}}", "provisioner","policy") - will set the value of config["provisioner"]["policy"] in the yaml file if - it does not exists an error will be printing that the value does not - exists. Alternatively you can use the . notation e.g. - set("{'project':{'fg82':[i0-i10]}}", "provisioner.policy") - """ - element = self - - if keys is None: - return self - - if '.' in keys[0]: - keys = keys[0].split(".") - - nested_str = ''.join(["['{0}']".format(x) for x in keys]) - # Safely evaluate an expression to see if it is one of the Python - # literal structures: strings, numbers, tuples, lists, dicts, booleans, - # and None. Quoted string will be used if it is none of these types. - try: - ast.literal_eval(str(value)) - converted = str(value) - except ValueError: - converted = "'" + str(value) + "'" - exec("self" + nested_str + "=" + converted) - return element - - def _update(self, keys, value): - """Updates the selected key with the value - - Args: - keys (str): key names e.g. cloudmesh.server.loglevel - value (str): value to set - """ - return self.set(value, keys) - - def attribute(self, keys): - """ - TODO: document this method - - :param keys: - """ - if self['meta']['prefix'] is None: - k = keys - else: - k = self['meta']['prefix'] + "." + keys - return self.get(k) - - -if __name__ == "__main__": - # TODO: etc not supported - # need to create copy and work with that - config = ConfigDict({"a": "1", "b": {"c": 3}}, - prefix="cloudmesh.debug", - filename="./etc/cloudmesh_debug.yaml") - - print("PPRINT") - print(70 * "=") - pprint(config) - - print("PRINT") - print(70 * "=") - print(config.pprint()) - print(config.json()) - - print(70 * "=") - print("A =", config["a"]) - config.write(config_file("/d.yaml"), output="dict") - config.write(config_file("/j.yaml"), output="json") - config.write(config_file("/y.yaml"), output="yaml") - - # this does not work - # config.write(config_file("/print.yaml"), output="print") - - print("mongo.path GET =", config.get("cloudmesh.server.mongo.path")) - print("mongo.path ATTRIBUTE =", config.attribute("mongo.path")) - - print("get A =", config.get("a")) - - print("wrong mongo.path ATTRIBUTE =", config.attribute("mongo.path.wrong")) - print("wrong mongo.path GET =", - config.get("cloudmesh.server.mongo.path.wrong")) - - # print config["dummy"] - # config["x"] = "2" - # print config["x"] - # print config.x diff --git a/deprecated/ConfigDict.py b/deprecated/ConfigDict.py deleted file mode 100644 index f2324bc5..00000000 --- a/deprecated/ConfigDict.py +++ /dev/null @@ -1,522 +0,0 @@ -""" -Manage the cloudmesh.yaml file -""" -import json -import os.path -import sys -from collections import OrderedDict - -from ruamel import yaml - -from cloudmesh.common.BaseConfigDict import BaseConfigDict -from cloudmesh.common.console import Console -from cloudmesh.common.todo import TODO -from cloudmesh.common.util import backup_name -from cloudmesh.common.util import path_expand -from pathlib import Path - - -def custom_print(data_structure, indent, attribute_indent=4): - """ - prints the data structure at a given level. This includes dicts and - ordered dicts - - :param data_structure: - :param indent: - :param attribute_indent: - :return: - """ - for key, value in data_structure.items(): - print("\n%s%s:" % (' ' * attribute_indent * indent, str(key)), end=' ') - if isinstance(value, OrderedDict): - custom_print(value, indent + 1) - elif isinstance(value, dict): - custom_print(value, indent + 1) - else: - print("%s" % (str(value)), end=' ') - - -# noinspection PyPep8Naming -def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **keywords): - """ - writes the dict into an ordered yaml. - - :param data: The ordered dict - :param stream: the stream - :param Dumper: the dumper such as yaml.SafeDumper - """ - - # noinspection PyClassHasNoInit - class OrderedDumper(Dumper): - """ - A helper class to print an ordered dict - """ - pass - - def _dict_representer(dumper, data): - return dumper.represent_mapping( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - data.items()) - - OrderedDumper.add_representer(OrderedDict, _dict_representer) - return yaml.dump(data, stream, OrderedDumper, **keywords) - - -# noinspection PyPep8Naming -def dprint(OD, mode='dict', s="", indent=' ' * 4, level=0): - """ - a recursive dict printer method that adds indentations - TODO: needs better explanation and test example - - :param OD: the ordered dict - :param mode: the mode is dict - :param s: TODO - :param indent: the indentation characters. default is 4 - :param level: the level - :return: - """ - - def is_number(s): - """ - checks if the type of s is a float - :param s: - :return: - """ - try: - float(s) - return True - except ValueError: - return False - - def fstr(s): - """ - return a string or number - :param s: teh string or number - :return: - """ - return s if is_number(s) else '"%s"' % s - - if mode != 'dict': - kv_tpl = '("%s", %s)' - ST = 'OrderedDict([\n' - END = '])' - else: - kv_tpl = '"%s": %s' - ST = '{\n' - END = '}' - for i, k in enumerate(OD.keys()): - if type(OD[k]) in [dict, OrderedDict]: - level += 1 - s += (level - 1) * indent + kv_tpl % ( - k, ST + dprint(OD[k], mode=mode, - indent=indent, level=level) + ( - level - 1) * indent + END) - level -= 1 - else: - s += level * indent + kv_tpl % (k, fstr(OD[k])) - if i != len(OD) - 1: - s += "," - s += "\n" - return s - - -class Config(object): - """ - Manage configuration files in yaml format - """ - - @classmethod - def check_file_for_tabs(cls, filename, verbose=True): - """identifies if the file contains tabs and returns True if it - does. It also prints the location of the lines and columns. If - verbose is set to False, the location is not printed. - - :param verbose: if true prints issues - :param filename: the filename - :type filename: str - :rtype: True if there are tabs in the file - """ - filename = path_expand(filename) - file_contains_tabs = False - - with open(filename, 'r') as f: - lines = f.read().splitlines() - - line_no = 1 - for line in lines: - if "\t" in line: - file_contains_tabs = True - location = [ - i for i in range(len(line)) if line.startswith('\t', i)] - if verbose: - print("Tab found in line", line_no, "and column(s)", - location) - line_no += 1 - return file_contains_tabs - - @classmethod - def path_expand(cls, path): - """ - expands the path while replacing environment variables, `./,` and `~/` - - :param path: the path to be expanded - :type path: string - :return:the new path - :rtype: string - """ - current_dir = "." + os.path.sep - if path.startswith(current_dir): - cwd = str(os.getcwd()) - path = path.replace(current_dir, cwd, 1) - location = os.path.expandvars(os.path.expanduser(path)) - return location - - @classmethod - def find_file(cls, filename, load_order=None, verbose=False): - """ - find the specified file in the list of directories that are given in the - array load_order - - :param filename: the file name - :type filename: str - :param load_order: an array with path names in with the filename is looked for. - :type load_order: list of str - :param verbose: - :type verbose: bool - :return: file name if successful - :rtype: string if the file exists or None otherwise - """ - if load_order is None: - load_order = [".", os.path.join("~", ".cloudmesh")] - for path in load_order: - name = Config.path_expand(path + os.path.sep + filename) - if verbose: - print("try finding file", name) - if os.path.isfile(name): - if verbose: - print("Found File", name) - return name - return None - - -class ConfigDict(object): - """ - Manages the content of cloudmesh.yaml as dict - """ - __shared_state = {} - versions = ['4.1'] - data = {} - filename = None - - def __init__(self, - filename, - load_order=None, - verbose=False, - etc=False, - reload=False): - - """ - Creates a dictionary from a yaml configuration file - while using the filename to load it in the specified load_order. - The load order is an array of paths in which the file is searched. - By default the load order is set to . and ~/.cloudmesh - - :param filename: the filename - :type filename: string - :param load_order: an array with path names in with the filename is looked for. - :type load_order: list of strings - :return: an instance of ConfigDict - :rtype: ConfigDict - """ - - self.__dict__ = self.__shared_state - - if ConfigDict.data != {} and not reload: - return - - # print ("INIT CONFIGDICT", filename, load_order, verbose, etc) - self.data = None - if etc: - # import cloudmesh_client.etc - # load_order = [os.path.dirname(cloudmesh_client.etc.__file__)] - print("etc not supported") - sys.exit(1) - - if load_order is None: - self.load_order = [".", ".cloudmesh", os.path.join("~", ".cloudmesh")] - else: - self.load_order = load_order - - for path in self.load_order: - name = Path(Config.path_expand(os.path.join(path, filename))) - if verbose: - print("try Loading ConfigDict", name) - if os.path.isfile(str(name)): - if verbose: - print("Loading ConfigDict", name) - self.load(str(name)) - ConfigDict.filename = str(name) - ConfigDict.data = self.data - return - - # Create default yaml file - raise ValueError( - "Could not find file {:} in {:}".format(filename, self.load_order)) - - def load(self, filename): - """ - loads the configuration from the yaml filename - - :param filename: - :type filename: string - :return: - """ - # print ("LOAD CONFIGDICT", filename) - - self.data = BaseConfigDict(filename=Config.path_expand(filename)) - try: - version = str(self.data["meta"]["version"]) - if version not in self.versions: - Console.error("The yaml file version must be {}".format( - ', '.join(self.versions))) - sys.exit(1) - except Exception as e: - Console.error( - f"Your yaml file {filename} is not up to date.", - traceflag=False) - Console.error(e.message, traceflag=False) - sys.exit(1) - # return self.data - - def write(self, filename=None, output="dict", attribute_indent=4): - """ - This method writes the dict into various output formats. This includes a dict, - json, and yaml - - :param filename: the file in which the dict is written - :param output: is a string that is either "dict", "json", "yaml" - :param attribute_indent: character indentation of nested attributes in - """ - if filename is not None: - location = path_expand(filename) - else: - location = self['meta']['location'] - - # with open('data.yml', 'w') as outfile: - # outfile.write( yaml.dump(data, default_flow_style=True) ) - - # Make a backup - self.make_a_copy(location) - - f = os.open(location, os.O_CREAT | os.O_TRUNC | - os.O_WRONLY, stat.S_IRUSR | stat.S_IWUSR) - if output == "json": - os.write(f, self.json()) - elif output in ['yml', 'yaml']: - # d = dict(self) - # os.write(f, yaml.dump(d, default_flow_style=False)) - os.write(f, ordered_dump(OrderedDict(self), - Dumper=yaml.SafeDumper, - default_flow_style=False, - indent=attribute_indent)) - elif output == "print": - os.write(f, str(custom_print(self, attribute_indent))) - else: - os.write(f, self.dump()) - os.close(f) - - def make_a_copy(self, location=None): - """ - Creates a backup of the file specified in the location. The backup - filename appends a .bak.NO where number is a number that is not yet - used in the backup directory. - TODO: This function should be moved to another file maybe XShell - - :param location: the location of the file to be backed up - """ - import shutil - destination = backup_name(location) - shutil.copyfile(location, destination) - - def save(self, filename=None): - """ - saves the configuration in the given filename, - if it is none the filename at load time is used. - - :param filename: the file name - :type filename: string - :return: - """ - content = self.data.yaml() - with open(Config.path_expand(ConfigDict.filename), 'w') as f: - f.write(content) - - def __setitem__(self, item, value): - """ - sets an item with the given value while using . formatted keys - set('a.b.c", value) - - :param item: - :type item: - :param value: - :type value: - :return: - """ - keys = None - if "." in item: - keys = item.split(".") - d = self.data - for i in keys[:-1]: - d = d[i] - d[keys[-1]] = value - else: - self.data[item] = value - self.save() - - def __getitem__(self, item): - """ - gets an item form the dict. The key is . separated - use it as follows get("a.b.c") - - :param item: - :type item: - :return: - """ - if "." in item: - keys = item.split(".") - else: - return self.data[item] - element = self.data[keys[0]] - for key in keys[1:]: - element = element[key] - return element - - def __str__(self): - """ - returns the dict in yaml format - - :return: returns the yaml output of the dict - :rtype: string - """ - return self.data.yaml() - - @property - def yaml(self): - """ - returns the dict in yaml format - - :return: returns the yaml output of the dict - :rtype: string: - """ - return self.data.yaml() - - def info(self): - """ - prints out the dict type and its content - """ - print(type(self.data)) - print(self.data) - - @property - def json(self, start=None): - """ - returns the dict in json format - - :param start: start key in dot notation - - :return: json string version - :rtype: string - """ - if start is not None: - data = self.data[start] - return json.dumps(self.data, indent=4) - - @classmethod - def check(cls, filename): - """ - checks the filename if it is syntactically correct and does not - include tabs - - :param filename: - :type filename: string - :return: - """ - TODO.implement() - - # noinspection PyPep8Naming - @classmethod - def getUser(cls, cloud): - """ - gets the username for a specified cloud. - TODO: works currently only for opensatck. - - :param cloud: the name of the cloud - :return: - """ - try: - config = d = ConfigDict("cloudmesh.yaml") - - d = ConfigDict("cloudmesh.yaml") - - # - # bug: cloud is none when adding a group - # - - config = d["cloudmesh"]["clouds"][cloud] - credentials = config["credentials"] - cloud_type = config["cm_type"] - - if cloud_type == "openstack": - return credentials["OS_USERNAME"] - else: - raise ValueError("getUser for this cloud type not yet " - "supported: {}".format(cloud)) - except Exception as ex: - Console.error("problem getting user") - - -# noinspection PyPep8Naming -def Username(): - """ - returns the username as defined in the profile - - :return: - """ - d = ConfigDict("cloudmesh.yaml") - - if "user" not in d["cloudmesh"]["profile"]: - raise RuntimeError("Profile username is not set in yaml file.") - - user = d["cloudmesh"]["profile"]["user"] - return user - - -def main(): - """ - TODO: A test which should actually be moved into a pytest - """ - d = ConfigDict("cloudmesh.yaml") - print(d, end='') - d.info() - - print(d["meta"]) - print(d["meta.kind"]) - print(d["meta"]["kind"]) - - # this does not yet work - d.data["cloudmesh"]["profile"]["firstname"] = 'ABC' - print(d) - d.save() - - import os - - os.system("cat cmd3.yaml") - - print(d.json) - print(d.filename) - print("YAML") - print(d.yaml) - - -if __name__ == "__main__": - main() diff --git a/deprecated/config.py b/deprecated/config.py deleted file mode 100644 index e22d3829..00000000 --- a/deprecated/config.py +++ /dev/null @@ -1,31 +0,0 @@ -import os - -from cloudmesh.common.dotdict import dotdict -from cloudmesh.common.util import path_expand -from cloudmesh.common.util import readfile -from ruamel import yaml -from pathlib import Path -from cloudmesh.common.base import Base - -class Config(object): - def __init__(self, filename=None): - self.filename = filename - base = Base() - if self.filename is None: - self.filename = Path(base.config) - content = readfile(self.filename) - self.data = yaml.load(content, Loader=yaml.RoundTripLoader) - - def cloud(self, name): - return self.data["cloudmesh"]["clouds"][name] - - def credentials(self, name): - return dotdict(self.cloud(name)["credentials"]) - - -''' -if __name__ == "__main__": - from pprint import pprint - c = Config() - pprint(c.credentials("jet")) -''' diff --git a/deprecated/deprecated_stopwatch_mllog.py b/deprecated/deprecated_stopwatch_mllog.py deleted file mode 100644 index 2d0e8553..00000000 --- a/deprecated/deprecated_stopwatch_mllog.py +++ /dev/null @@ -1,447 +0,0 @@ -############################################################### -# pytest -v --capture=no tests/test_stopwatch_mllog.py -# pytest -v --capture=no tests/test_stopwatch_mllog.py::Test_stopwatch.test_001 -# pytest -v tests/test_stopwatch_mllog.py -############################################################### - -from typing import Dict, Any -import contextlib -import json -import logging -import os -import time -from io import StringIO # Python 3 -import pathlib -import sys -import tempfile -from pprint import pprint - -import yaml - -from cloudmesh.common.Shell import Shell -from cloudmesh.common.util import readfile - -import pytest -from cloudmesh.common.StopWatch import StopWatch - -from cloudmesh.common.util import HEADING - -# Sample taken from https://github.com/mlcommons/logging/blob/master/mlperf_logging/mllog/constants.py -mllog_constants=dict( - DEFAULT_LOGGER_NAME = "mllog_default", - DEFAULT_NAMESPACE = "", - # Constant values - log event type - INTERVAL_END = "INTERVAL_END", - INTERVAL_START = "INTERVAL_START", - POINT_IN_TIME = "POINT_IN_TIME", - # Constant values - submission division - CLOSED = "closed", - OPEN = "open", -) - -#benchmark_constant = [ -# -#] - -benchmark_config = """ -benchmark: - name: Earthquake - user: Gregor von Laszewski - e-mail: laszewski@gmail.com - organisation: University of Virginia - division: BII - status: success - platform: rivanna - badkey: ignored -""".strip() - -# -# @contextlib.contextmanager -# def yaml_generator(filename, prefix="general", flat=False): -# if flat: -# mllog_sampleyaml = f""" -# name: ignored -# {prefix}.name: Earthquake -# {prefix}.user: Gregor von Laszewski -# {prefix}.e-mail: laszewski@gmail.com -# {prefix}.organisation: University of Virginia -# {prefix}.division: BII -# {prefix}.status: submission -# {prefix}.platform: rivanna -# {prefix}.badkey: ignored -# followon: ignored -# """ -# else: -# mllog_sampleyaml=""" -# name: ignored -# benchmark: -# name: Earthquake -# user: Gregor von Laszewski -# e-mail: laszewski@gmail.com -# organisation: University of Virginia -# division: BII -# status: submission -# platform: rivanna -# badkey: ignored -# followon: ignored -# """ -# my_tempdir = tempfile.gettempdir() -# temppath = pathlib.Path(my_tempdir) / filename -# with open(temppath, 'wb') as f: -# f.write(mllog_sampleyaml.encode('utf-8')) -# try: -# yield temppath -# finally: -# os.remove(temppath) -# -# -# def convert_mllog_to_object(log_stream: StringIO): -# """Utility method to transform the mlperf logging strings into a python object. -# -# :param log_stream: An io-like object that contains a stream of newline split json strings -# :type log_stream: StringIO -# -# :returns: a list of python dictionaries representing the logging events. -# :rtype: list(dict) -# """ -# obj = list() -# text = log_stream.getvalue().replace(":::MLLOG ", "") -# for x in text.split("\n"): -# if x != "": -# try: -# obj.append(json.loads(x)) -# except json.JSONDecodeError as e: -# pass -# return obj -# -# -# @contextlib.contextmanager -# def intercept_mllogger(filename=None): -# """A context function used to capture logging events in mlperf. -# This method does so by directly registering a stream object to the logging -# module. -# -# :yields: a StringIO object that captures the StopWatch mllog events. -# """ -# StopWatch.activate_mllog(filename=filename) -# temp_out = StringIO('') -# stream_handler = logging.StreamHandler(temp_out) -# StopWatch.mllogger.logger.addHandler(stream_handler) -# try: -# yield temp_out -# finally: -# StopWatch.mllogger.logger.removeHandler(stream_handler) -# - -@pytest.mark.incremental -class Test_Printer: - - def test_check_loading(self): - Shell.rm("cloudmesh_mllog.log") - StopWatch.activate_mllog() - StopWatch.mllogger.event("test", "1", stack_offset=1) - content = readfile("cloudmesh_mllog.log").splitlines() - line = Shell.find_lines_with(content, what='"key": "test"')[0] - print (line) - assert "cloudmesh-common/tests/test_stopwatch_mllog.py" in line - assert '"value": "1"' in line - assert '"value": "1"' in line - - def test_stopwatch_activate_mllog(self): - HEADING() - Shell.rm("cloudmesh_mllog.log") - StopWatch.activate_mllog() - print(StopWatch.mllogging) - assert StopWatch.mllogging == True - - def test_stopwatch_1(self): - HEADING() - StopWatch.start("stopwatch sleep 1") - time.sleep(0.1) - StopWatch.stop("stopwatch sleep 1") - StopWatch.status("stopwatch sleep 1", True) - content = readfile("cloudmesh_mllog.log").splitlines() - _start = content[0] - _stop = content[1] - - print() - print (_start) - print (_stop) - - assert "INTERVAL_START" in _start - assert "INTERVAL_END" in _stop - assert "stopwatch sleep 1" in _start - assert "stopwatch sleep 1" in _stop - assert True - - def test_stopwatch_2(self): - HEADING() - StopWatch.start("stopwatch sleep 2") - time.sleep(0.1) - StopWatch.stop("stopwatch sleep 2") - StopWatch.status("stopwatch sleep 2", True) - content = readfile("cloudmesh_mllog.log").splitlines() - _start = content[2] - _stop = content[3] - - print() - print (_start) - print (_stop) - - assert "INTERVAL_START" in _start - assert "INTERVAL_END" in _stop - assert "stopwatch sleep 2" in _start - assert "stopwatch sleep 2" in _stop - assert True - - def test_stopwatch_loop_sum(self): - HEADING() - cumulate = False - dt = 0.1 - n = 10 - for i in range(0,n): - StopWatch.start("stopwatch loop") - time.sleep(dt) - StopWatch.stop("stopwatch loop") - StopWatch.status("stopwatch loop", True) - t = StopWatch.get("stopwatch loop") - print (t) - assert t >= dt - - t = StopWatch.sum("stopwatch loop", digits=4) - - print (t) - - assert t >= n * dt - - def test_stopwatch_loop_individual(self): - HEADING() - cumulate = False - dt = 0.1 - n = 10 - for i in range(0,n): - StopWatch.start(f"stopwatch loop {i}") - time.sleep(dt) - StopWatch.stop(f"stopwatch loop {i}") - StopWatch.message(f"stopwatch loop {i}", i) - StopWatch.status(f"stopwatch loop {i}", True) - t = StopWatch.get(f"stopwatch loop {i}") - print (t) - assert t >= dt - - t = StopWatch.sum("stopwatch loop", digits=4) - - print (t) - - assert t >= n * dt - - def test_stopwatch_dict_event(self): - HEADING() - data = {"a": 1} - t = StopWatch.event("stopwtch dict event", values=data) - content = readfile("cloudmesh_mllog.log").splitlines()[-1] - - print (content) - assert "{'a': 1}" in content - assert '"key": "stopwtch dict event"' in content - - def test_stopwatch_organization(self): - HEADING() - data = yaml.safe_load(benchmark_config) - StopWatch.organization_mllog(**data) - - content = readfile("cloudmesh_mllog.log").splitlines() - print ("---") - pprint (content) - - content = readfile("cloudmesh_mllog.log") - assert '"namespace": "cloudmesh"' in content - assert '"event_type": "POINT_IN_TIME", "key": "submission_poc_name"' in content - assert '"value": "Gregor von Laszewski"' in content - assert '"event_type": "POINT_IN_TIME", "key": "submission_poc_email"' in content - assert '"value": "laszewski@gmail.com"' in content - assert '"event_type": "POINT_IN_TIME", "key": "submission_org"' in content - assert '"value": "University of Virginia"' in content - assert '"event_type": "POINT_IN_TIME", "key": "submission_division"' in content - assert '"value": "BII"' in content - assert '"event_type": "POINT_IN_TIME", "key": "submission_status"' in content - assert '"value": "submission"' in content - assert '"event_type": "POINT_IN_TIME", "key": "submission_platform"' in content - assert '"value": "rivanna"' in content - - def test_stopwatch_organization(self): - HEADING() - data = yaml.safe_load(benchmark_config) - StopWatch.organization_mllog(**data) - - content = readfile("cloudmesh_mllog.log").splitlines() - print("---") - pprint(content) - - content = readfile("cloudmesh_mllog.log") - - def test_manual_test(self): - HEADING() - Shell.rm("cloudmesh_mllog.log") - StopWatch.clear() - # this code is in a manula README-mlcommons.md - - # define the organization - - config = """ - benchmark: - name: Earthquake - user: Gregor von Laszewski - e-mail: laszewski@gmail.com - organisation: University of Virginia - division: BII - status: success - platform: rivanna - badkey: ignored - """.strip() - - submitter = yaml.safe_load(config) - - # activate the MLcommons logger - StopWatch.activate_mllog() - - StopWatch.organization_mllog(**submitter) - - # save an event with a value - StopWatch.event("test", values="1") - - # save a dict in an event - data = {"a": 1} - StopWatch.event("stopwtch dict event", values=data) - - # start a timer - StopWatch.start("stopwatch sleep") - - # do some work - time.sleep(0.1) - - # stop the timer - StopWatch.stop("stopwatch sleep") - - # print the table - StopWatch.benchmark(tag="Earthquake", node="summit", user="Gregor", version="0.1") - - #StopWatch.log_constant(**log) - - - - # def test_stopwatch_log_event(self): - # HEADING() - # data = { "a": 1 } - # with intercept_mllogger() as log_stream: - # StopWatch.log_event(values=data) - # log_text = log_stream.getvalue() - # assert ':::MLLOG' in log_text, 'logger must have mllog entries when activated' - # - # log_object = convert_mllog_to_object(log_stream) - # assert ( - # all(list(map(lambda x: x['namespace'] == "cloudmesh", log_object))), - # "Logger must have cloudmesh namespace." - # ) - # - # assert ( - # any(list(map(lambda x: x['key'] == "mllog-event-values", log_object))), - # "Logger event keys must apply the key label across all logs" - # ) - # - # for x in log_object: - # if x['event_type'] == "POINT_IN_TIME": - # assert isinstance(x['value'], str), "Value must be converted to string." - # assert x['value'] == str(data), "Event data must capture passed values in string format" - # assert len(log_object) == 1, "MLPerf logging must not trigger start/stop entries when logging event" - # - # def test_stopwatch_log_constant(self): - # HEADING() - # data = {"a": 1} - # log = {"DEFAULT_LOGGER_NAME": "testing default logger name"} - # with intercept_mllogger() as log_stream: - # StopWatch.log_constant(**log) - # log_text = log_stream.getvalue() - # assert ':::MLLOG' in log_text, 'logger must have mllog entries when activated' - # log_object = convert_mllog_to_object(log_stream) - # for x in log_object: - # if x['event_type'] == "POINT_IN_TIME": - # assert ( - # x['key'] == mllog_constants["DEFAULT_LOGGER_NAME"], - # "Logged key must match string representation in mlperf_logging" - # ) - # - # def test_stopwatch_log_evalblock(self): - # HEADING() - # data = { "a": 1, "b": 5 } - # with intercept_mllogger() as log_stream: - # StopWatch.start("Test Notebook", mllog_key="BLOCK_START") - # StopWatch.log_event(MY_TEST_KEY=data) - # time.sleep(2) - # StopWatch.stop("Test Notebook", values=data, mllog_key="BLOCK_STOP") - # log_text = log_stream.getvalue() - # assert ':::MLLOG' in log_text, 'logger must have mllog entries when activated' - # - # log_object = convert_mllog_to_object(log_stream) - # - # assert len(log_object) == 3, "Multiple log entires should be independent events" - # - # assert ( - # log_object[0]["event_type"] == "INTERVAL_START" and \ - # log_object[1]["event_type"] == "POINT_IN_TIME" and \ - # log_object[2]["event_type"] == "INTERVAL_END", - # "Events must occur in sequence." - # ) - # - # assert ( - # log_object[0]["key"] == "block_start", - # "mllog_key constant references should dereference when logged." - # ) - # assert ( - # log_object[1]["key"] == "mllog-event-MY_TEST_KEY", - # "Log events that do not map to mllog constants should be prefixed with 'mllog-event-'" - # ) - # assert ( - # log_object[1]["value"] == str(data), - # "Logged events should persist data in string format" - # ) - # assert ( - # log_object[2]["value"] == str(data), - # "Events must support the persistance of values." - # ) - # assert ( - # log_object[0]["value"] is None, - # "Events must not require values." - # ) - # - # def test_stopwatch_log_evalblock(self): - # HEADING() - # with intercept_mllogger() as log_stream: - # StopWatch.start("Test Notebook", mllog_key="BLOCK_START") - # StopWatch.stop("Test Notebook", mllog_key="BLOCK_STOP") - # log_text = log_stream.getvalue() - # - # def test_stopwatch_organization_mllog(self): - # for method in (True, False): - # with yaml_generator('test.yml', flat=method) as f: - # with intercept_mllogger() as log_stream: - # StopWatch.start("Test Notebook", mllog_key="BLOCK_START") - # StopWatch.organization_mllog(f, prefix_="general", flatdict_=method) - # StopWatch.stop("Test Notebook", mllog_key="BLOCK_STOP") - # log_text = log_stream.getvalue() - # assert ':::MLLOG' in log_text, 'logger must have mllog entries when activated' - # log_object = convert_mllog_to_object(log_stream) - # assert log_object[1]['key'] == "submission_benchmark" and log_object[1]['value'] == "Earthquake" and \ - # log_object[2]['key'] == "submission_poc_name" and log_object[2]['value'] == "Gregor von Laszewski" and \ - # log_object[3]['key'] == "submission_poc_email" and log_object[3]['value'] == "laszewski@gmail.com" and \ - # log_object[4]['key'] == "submission_org" and log_object[4]['value'] == "University of Virginia" and \ - # log_object[5]['key'] == "submission_division" and log_object[5]['value'] == "BII" and \ - # log_object[6]['key'] == "submission_status" and log_object[6]['value'] == "submission" and \ - # log_object[7]['key'] == "submission_platform" and log_object[7]['value'] == "rivanna" - # assert not any([obj['value'] == "ignored" for obj in log_object]) - # - # - - def test_benchmark(self): - HEADING() - StopWatch.benchmark(sysinfo=False, tag="cc-db", user="test", node="test") diff --git a/deprecated/etc/__init__.py b/deprecated/etc/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/deprecated/etc/cloudmesh.yaml b/deprecated/etc/cloudmesh.yaml deleted file mode 100644 index 1231adec..00000000 --- a/deprecated/etc/cloudmesh.yaml +++ /dev/null @@ -1,227 +0,0 @@ -meta: - version: 4.1 - kind: clouds -cloudmesh: - profile: - firstname: TBD - lastname: TBD - email: TBD - user: TBD - github: - username: TBD - portal: - location: TBD - browser: firefox - comet: - username: TBD - active: production - endpoints: - dev: - nucleus_base_url: https://comet-nucleus-dev.sdsc.edu/nucleus - api_version: v1 - auth_provider: apikey - userpass: - username: TBD - password: TBD - apikey: - api_key: TBD - api_secret: TBD - production: - nucleus_base_url: https://comet-nucleus.sdsc.edu/nucleus - api_version: v1 - auth_provider: apikey - userpass: - username: TBD - password: TBD - apikey: - api_key: TBD - api_secret: TBD - hpc: - experiment: - name: gregor-00000 - active: - - comet - - juliet - clusters: - india: - cm_heading: India HPC CLuster - cm_host: india - cm_label: indiahpc - cm_type: slurm - cm_type_version: 14.11.9 - credentials: - username: TBD - project: None - default: - queue: delta - experiment_dir: /N/u/{username}/experiment - prefix: {username} - comet: - cm_heading: Comet HPC CLuster - cm_host: comet - cm_label: comethpc - cm_type: slurm - cm_type_version: 14.11.9 - credentials: - username: TBD - project: None - default: - queue: debug - experiment_dir: /home/{username}/experiment - prefix: {username} - active: - - kilo - clouds: - kilo: - cm_heading: India OpenStack, Kilo - cm_host: india - cm_label: kilo - cm_type: openstack - cm_type_version: kilo - cm_openrc: ~/.cloudmesh/clouds/india/kilo/openrc.sh - credentials: - OS_AUTH_URL: https://kilo.futuresystems.org:5000/v3 - OS_PASSWORD: TBD - OS_TENANT_NAME: TBD - OS_USERNAME: TBD - OS_PROJECT_DOMAIN_ID: default - OS_USER_DOMAIN_ID: default - OS_PROJECT_NAME: TBD - OS_IMAGE_API_VERSION: 2 - OS_VOLUME_API_VERSION: 2 - default: - flavor: m1.small - image: Ubuntu-14.04-64 - chameleon: - cm_heading: Chameleon - cm_host: chameleoncloud.org - cm_label: chameleon - cm_type: openstack - cm_type_version: liberty - credentials: - OS_AUTH_URL: https://openstack.tacc.chameleoncloud.org:5000/v3 - OS_PASSWORD: TBD - OS_TENANT_NAME: CH-818144 - OS_TENANT_ID: CH-818144 - OS_PROJECT_NAME: CH-818144 - OS_PROJECT_DOMAIN_ID: default - OS_USER_DOMAIN_ID: default - OS_USERNAME: TBD - OS_VERSION: liberty - OS_REGION_NAME: RegionOne - default: - flavor: m1.small - image: Ubuntu-Server-14.04-LTS - jetstream: - cm_heading: Jetstream - cm_host: http://jetstream-cloud.org/ - cm_label: jetstream - cm_type: openstack - cm_type_version: kilo - credentials: - OS_PROJECT_DOMAIN_NAME: tacc - OS_USER_DOMAIN_NAME: tacc - OS_PROJECT_NAME: TBD - OS_TENANT_NAME: TBD - OS_USERNAME: TBD - OS_PASSWORD: TBD - OS_AUTH_URL: https://jblb.jetstream-cloud.org:35357/v3 - OS_IDENTITY_API_VERSION: 3 - OS_VERSION: kilo - default: - flavor: m1.small - image: Ubuntu-Server-14.04-LTS - cybera-c: - cm_heading: Cybera Calgary OpenStack - cm_host: cybera - cm_label: cybera-c - cm_type: openstack - cm_type_version: kilo - credentials: - OS_AUTH_URL: TBD - OS_TENANT_ID: TBD - OS_TENANT_NAME: TBD - OS_PROJECT_NAME: TBD - OS_USERNAME: TBD - OS_PASSWORD: TBD - OS_REGION_NAME: Calgary - default: - flavor: m1.small - image: Ubuntu 14.04 - cybera-e: - cm_heading: Cybera Edmonton OpenStack - cm_host: cybera - cm_label: kilo - cm_type: openstack - cm_type_version: kilo - credentials: - OS_AUTH_URL: https://keystone-yyc.cloud.cybera.ca:5000/v2.0 - OS_TENANT_ID: TBD - OS_TENANT_NAME: TBD - OS_PROJECT_NAME: TBD - OS_USERNAME: TBD - OS_PASSWORD: TBD - OS_REGION_NAME: Edmonton - default: - flavor: m1.small - image: Ubuntu 14.04 - aws: - cm_heading: Amazon Cloud, AWS - cm_host: aws.amazon.com - cm_label: aws - cm_type: ec2 - cm_type_version: null - credentials: - EC2_ACCESS_KEY: TBD - EC2_SECRET_KEY: TBD - keyname: TBD - userid: TBD - default: - flavor: t1.micro - image: ami-d85e75b0 - location: us-east-1 - chameleon-ec2: - cm_heading: Chameleon, EC2 - cm_host: chameleoncloud.org - cm_label: chameleon_ec2 - cm_type: ec2 - cm_type_version: ec2 - credentials: - EC2_ACCESS_KEY: TBD - EC2_SECRET_KEY: TBD - keyname: TBD_not_used - userid: TBD_not_used - EC2_URL: https://openstack.tacc.chameleoncloud.org:8773/services/Cloud - EC2_USER_ID: TBD - EC2_PRIVATE_KEY: ~/.cloudmesh/clouds/chameleon/TBD/pk.pem - EC2_CERT: ~/.cloudmesh/clouds/chameleon/TBD/cert.pem - NOVA_CERT: ~/.cloudmesh/clouds/chameleon/TBD/cacert.pem - EUCALYPTUS_CERT: ~/.cloudmesh/clouds/chameleon/TBD/cacert.pem - default: - flavor: m1.small - image: Ubuntu-Server-14.04-LTS - azure: - cm_heading: Microsoft Azure Virtual Machines - cm_host: windowsazure.com - cm_label: azure - cm_type: azure - cm_type_version: null - credentials: - managementcertfile: TBD - servicecertfile: TBD - subscriptionid: TBD - thumbprint: TBD - default: - username: TBD - password: TBD - flavor: Basic_A1 - image: b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-12_04_5-LTS-amd64-server-20160315-en-us-30GB - location: Central US - system: - data: ~/.cloudmesh/cloudmesh_inventory.yaml - console_color: true - logging: - file: ~/.cloudmesh/cloudmesh.log - level: DEBUG - plugins: - - vbox: cloudmesh_vagrant cloudmesh_vagrant.cm_vbox.do_vbox diff --git a/deprecated/old_init.py b/deprecated/old_init.py deleted file mode 100644 index 493cf466..00000000 --- a/deprecated/old_init.py +++ /dev/null @@ -1,20 +0,0 @@ - -def create_cloudmesh_yaml(filename): - if not os.path.exists(filename): - path = os.path.dirname(filename) - if not os.path.isdir(path): - Shell.mkdir(path) - etc_path = os.path.dirname(__file__) - etc_file = os.path.join(etc_path, "etc", "cloudmesh.yaml") - to_dir = path_expand("~/.cloudmesh") - shutil.copy(etc_file, to_dir) - os.system("chmod -R go-rwx " + path_expand("~/.cloudmesh")) - Console.ok("~/.cloudmesh/cloudmesh.yaml created") - - -def setup_yaml(): - filename = path_expand("~/.cloudmesh/cloudmesh.yaml") - create_cloudmesh_yaml(filename) - - -setup_yaml() diff --git a/deprecated/test_configdict.py b/deprecated/test_configdict.py deleted file mode 100644 index 7652e72d..00000000 --- a/deprecated/test_configdict.py +++ /dev/null @@ -1,108 +0,0 @@ -############################################################### -# pip install .; pytest -v --capture=no tests/test_configdict..py::Test_configdict.test_001 -# pytest -v --capture=no tests/test_configdictr.py -# pytest -v tests/test_configdict.py -############################################################### - -import os - -from cloudmesh.common.ConfigDict import ConfigDict -from cloudmesh.common.util import HEADING -import pytest - -@pytest.mark.incremental -class Test_configdict: - - # noinspection PyPep8Naming - def tearDown(self): - pass - - def test_001_read(self): - HEADING("test if cloudmesh.yaml is loaded") - d = ConfigDict("cloudmesh.yaml", - verbose=True) - - assert d["cloudmesh"]["profile"]["firstname"] != "" - assert len(d["cloudmesh"]["clouds"]) > 0 - - """ - # - # DO NOT DO THIS TEST AS LOGIC OF AUTOCREATION HAS CHANGED - # - # try: - # d = ConfigDict("cloudmesh.yam", - # verbose=True) - # print("the file cloudmesh.yam should not exists") - # assert False - # except Exception as e: - # print (">>>>>>>", e) - # assert str(e).startswith("Could not find") - """ - - """ do not test for etc - def test_002_set(self): - HEADING("testing to set a value in the dict") - shutil.copy(self.etc_yaml, self.tmp_yaml) - d = ConfigDict("cloudmesh.yaml", - load_order=[self.tmp_dir], - verbose=True) - d["cloudmesh"]["profile"]["firstname"] = "Gregor" - d.save() - - d = ConfigDict("cloudmesh.yaml", - load_order=[self.tmp_dir], - verbose=True) - assert d["cloudmesh"]["profile"]["firstname"] == "Gregor" - """ - - def test_003_json(self): - HEADING("test if json is produced") - d = ConfigDict("cloudmesh.yaml", - verbose=True) - - assert d.json.startswith('{') - - try: - assert not isinstance(d.json, str) - print("json should be string") - assert False - except Exception as e: - assert isinstance(d.json, str) - - def test_004_yaml(self): - - HEADING("test if yaml is produced") - d = ConfigDict("cloudmesh.yaml", - verbose=True) - result = d.yaml - - try: - assert result.startswith("meta") - except Exception as e: - print("not valid yaml file.") - assert False - - -""" def main(): - d = ConfigDict("cmd3.yaml") - print (d, end='') - print(Printer.write(self.cm.info())) - - - print (d["meta"]) - print (d["meta.kind"]) - print (d["meta"]["kind"]) - - # this does not yet work - print (d) - d.save() - - import os - os.system("cat cmd3.yaml") - - print(d.json) - print(d.filename) - -if __name__ == "__main__": - main() -""" diff --git a/makefile-info.mk b/makefile-info.mk new file mode 100644 index 00000000..63efee57 --- /dev/null +++ b/makefile-info.mk @@ -0,0 +1,6 @@ +info: + @echo "=================================================" + @git remote show origin + @echo "=================================================" + @git shortlog -sne --all + @echo "================================================="