29 generate contributed defs

nxdl/README.md

@@ -15,7 +15,20 @@ The result of executing **nxdl_to_hdf5 -d applications** is the following which
     - A python script to create the definition using h5py is created in **exampledata/autogenerated_examples/nxdl/h5py**
     - A python script to create the definition using nexusformat is created in **exampledata/autogenerated_examples/nxdl/nexusformat**
 
-**Dependancies**
+## Contributed Definition testing
+
+A good way to stay on track while developing a new application definition is to use ```nxdl_to_hdf5.py```
+to try and generate your definition, if there are basic errors they will be reported, and if your
+definition generates an hdf5 file without errors there is a good chance that it will also pass ```cnxvalidate```.
+As with any other definition, a python script using both **h5py** and **nexusformat** modules will also be created
+to give you a tangible example of what is needed to create a compliant file for your definition.
+
+To generate your definition, save your definitions **.nxdl.xml** file to the 
+`nexusformat/definitions/contributed_definitions` folder, then generate the file by executing: 
+
+ `python nxdl_to_hdf5.py -d contributed_definitions`
+
+### **Dependancies**
 
 nxdl_to_hdf5 requires the following modules:
 
@@ -27,7 +40,7 @@ nxdl_to_hdf5 requires the following modules:
     - bs4
     - tinydb
 
-**Arguments and Usage:**
+### **Arguments and Usage:**
 ```     
 >python nxdl_to_hdf5.py --help
     usage: nxdl_to_hdf5.py [-h] [-f FILE | -d DIRECTORY]

nxdl/nxdl_to_hdf5.py

@@ -1,4 +1,4 @@
-from pathlib import Path
+import pathlib
 import os
 import pkg_resources
 import h5py
@@ -45,7 +45,8 @@
 
 def init_database():
     global db, query, tables_dct
-    if os.path.exists('db.json'):
+
+    if pathlib.Path('db.json').exists():
         # reset database to nothing
         if db is not None:
             db.close()
@@ -598,6 +599,12 @@ def get_nx_data_by_type(nx_type, dimensions=None, sym_dct={}):
             return (data)
         else:
             return (1)
+    elif (nx_type.find('NX_DIMENSIONLESS') > -1):
+        if (use_dims):
+            return (data)
+        else:
+            return (1)
+
 
 
 def get_units(nx_type):
@@ -617,11 +624,21 @@ def make_timestamp_now():
     t = datetime.datetime.now().isoformat()
     return (t)
 
+def fatal_error(msg):
+    '''
+    a fatal error has occurred, print message
+    '''
+    print('\tFATAL_ERROR: %s' % msg)
+
+
 def get_entry(nf):
     '''
     return the name of the entry in the file
     '''
     keys = list(nf.keys())
+    if len(keys) == 0:
+        #fatal_error('File does not contain NXentry group')
+        return(None, None)
     s1 = 'entry'
     for k in keys:
         if s1.casefold() == k.casefold():
@@ -782,24 +799,25 @@ def process_symbols(soup, sym_args_dct={}):
 def get_extending_class(fname, cls_nm, sym_args_dct={}, dct={}, docs=[], report_symbols_only=False):
     fparts = fname.split('\\')
     eclass_file = fname.replace(fparts[-1], '%s.nxdl.xml' % cls_nm)
-    if (not os.path.exists(eclass_file)):
+
+    if not pathlib.Path(eclass_file).exists():
         print('get_extending_class: XML file [%s] does not exist' % eclass_file)
         return({}, {}, {})
     print('extending with [%s]' % cls_nm)
     dct, syms, docs = get_xml_paths(eclass_file, sym_args_dct=sym_args_dct, dct=dct, docs=docs, report_symbols_only=report_symbols_only, allow_extend=True)
 
     return (dct, syms, docs)
 
-def get_xml_root(fname):
+def get_xml_root(fpath):
     '''
     takes the path to teh nxdl.xml file and returns a dict of element category lists of the entire structure
     '''
 
-    if(not fname.exists()):
-        print('XML file [%s] does not exist' % str(fname.absolute()))
+    if not fpath.exists():
+        print('XML file [%s] does not exist' % str(fpath))
         return(None)
 
-    infile = open(fname, "r")
+    infile = open(str(fpath), "r")
     contents = infile.read()
     infile.close()
     contents = contents.replace('xmlns="http://definition.nexusformat.org/nxdl/3.1"','')
@@ -823,13 +841,13 @@ def walk_extends_chain(fpath, root):
     main_clss = main_clss.replace('.nxdl.xml','')
     ext_lst = [main_clss]
     extends_clss = root.get('extends')
-    fpath = Path(str(fpath.absolute()).replace(main_clss, f'{extends_clss}'))
+    fpath = pathlib.Path(str(fpath.absolute()).replace(main_clss, f'{extends_clss}'))
     while extends_clss != 'NXobject':
         #(dct, syms, docs) =
         ext_lst.append(extends_clss)
         _root, _soup = get_xml_root(fpath)
         extends_clss = _root.get('extends')
-        fpath = Path(str(fpath.absolute()).replace(ext_lst[-1], f'{extends_clss}'))
+        fpath = pathlib.Path(str(fpath.absolute()).replace(ext_lst[-1], f'{extends_clss}'))
     ext_lst.reverse()
     return(ext_lst)
 
@@ -839,14 +857,14 @@ def get_xml_paths(fname, sym_args_dct={}, dct={}, docs=[], report_symbols_only=F
     '''
 
     fname = fname.replace('\\', '/')
-    fpath = Path(fname)
+    fpath = pathlib.Path(fname)
     _root, _soup = get_xml_root(fpath)
     if _root is None:
         return
     extends_lst = walk_extends_chain(fpath, _root)
     def_lst = []
     for ext_clss in extends_lst:
-        fpath = Path(str(fpath.absolute()).replace(fpath.name, f'{ext_clss}.nxdl.xml'))
+        fpath = pathlib.Path(str(fpath.absolute()).replace(fpath.name, f'{ext_clss}.nxdl.xml'))
         tables_dct['main'].insert({'filename': str(fpath.name)})
         _root, _soup = get_xml_root(fpath)
         def_lst.append(get_definition_details(_root, _soup))
@@ -1109,7 +1127,7 @@ def create_fields(nf, sym_dct={}, category=''):
         else:
             data = get_nx_data_by_type(_type, use_dim_dct_lst, sym_dct)
             if(data is None):
-                print('\t\tError: There is an issue with a non standard field for fieldname [%s]' % name)
+                print('\t\tError: There is an issue with a non standard field for fieldname [%s]:  ' % name)
                 return(False)
 
         # if name.find('depends_on'):
@@ -1343,19 +1361,19 @@ def print_nxsfrmt_close(class_nm):
     nxsfrmt_script_lst.append('root.save(\'%s.nxs\', \'w\')\n\n' % class_nm)
 
 
-def make_class_as_nf_file(clss_nm, dest_dir, symbol_dct={}):
+def make_class_as_nf_file(class_nm, dest_dir, symbol_dct={}):
     '''
     create an hdf5 file of the application definition (class_nm) in the specified destination directory
     '''
 
-    print('\texporting: [%s]' % clss_nm)
+    print('\texporting: [%s]' % class_nm)
     res = True
     category = def_dir = get_category()
 
-    if (not os.path.exists(dest_dir)):
+    if not pathlib.Path(dest_dir).exists():
         os.makedirs(dest_dir)
 
-    fpath = os.path.join(dest_dir, '%s.hdf5' % clss_nm)
+    fpath = str(pathlib.PurePath(dest_dir, '%s.hdf5' % class_nm))
     nf = h5py.File(fpath, 'w')
 
     sym_dct = {}
@@ -1424,6 +1442,9 @@ def make_class_as_nf_file(clss_nm, dest_dir, symbol_dct={}):
         _string_attr(nf, 'h5py_version', h5py.version.version, do_print=False)
         #_string_attr(nf, 'NEXUS_release_ver', rel_ver)
         entry_grp, entry_nm = get_entry(nf)
+        if entry_grp is None:
+            fatal_error('File does not contain an NXentry group')
+            return(None)
         #ensure the definition is correct
         entry_grp['definition'][()] = get_cur_def_name()
         _string_attr(nf, 'default', entry_nm)
@@ -1435,7 +1456,7 @@ def make_class_as_nf_file(clss_nm, dest_dir, symbol_dct={}):
                 _string_attr(nx_data_grp, 'signal', dset_nm)
                 _string_attr(nx_data_grp[dset_nm], 'signal', '1')
 
-        _dataset(nf, 'README', readme_string % (rel_ver, clss_nm), 'NX_CHAR', nx_units='NX_UNITLESS', dset={}, do_print=False)
+        _dataset(nf, 'README', readme_string % (rel_ver, class_nm), 'NX_CHAR', nx_units='NX_UNITLESS', dset={}, do_print=False)
 
         prune_extended_entries(nf)
 
@@ -1445,13 +1466,10 @@ def make_class_as_nf_file(clss_nm, dest_dir, symbol_dct={}):
         print('Failed exporting [%s]' % fpath)
         nf.close()
 
-
     print_script_versions(class_nm)
     print_script_close(class_nm)
 
-    write_script_file(class_nm)
-
-
+    write_script_files(class_nm)
 
     return(res)
 
@@ -1480,50 +1498,55 @@ def print_nxsfrmt_versions(class_nm):
     # nxsfrmt_script_lst.append('root.attrs[\'HDF5_Version\'] = h5py.version.hdf5_version')
     pass
 
-def write_script_file(class_nm):
-    write_h5py_script(os.path.join(os.getcwd(), '..', 'autogenerated_examples','nxdl', 'python_scripts','h5py'), class_nm, h5py_script_lst)
-    write_nxsfrmt_script(os.path.join(os.getcwd(), '..', 'autogenerated_examples','nxdl', 'python_scripts', 'nexusformat'), class_nm, nxsfrmt_script_lst)
+def write_script_files(class_nm):
+    mod_name = 'h5py'
+    write_script(mod_name,
+                 pathlib.PurePath(pathlib.os.getcwd(), '..', 'autogenerated_examples','nxdl', 'python_scripts', mod_name),
+                 class_nm,
+                 h5py_script_lst)
+    mod_name = 'nexusformat'
+    write_script(mod_name,
+                 pathlib.PurePath(mod_name, pathlib.os.getcwd(), '..', 'autogenerated_examples','nxdl', 'python_scripts', mod_name),
+                 class_nm,
+                 nxsfrmt_script_lst)
 
-def write_h5py_script(path, class_nm, script_lst):
-        if not os.path.exists(path):
-            os.makedirs(path)
-        f = open(os.path.join(path, 'ex_h5py_%s.py' % class_nm), 'w')
-        for l in script_lst:
-            f.write(l + '\n')
-        f.close()
-
-def write_nxsfrmt_script(path, class_nm, script_lst):
-    if not os.path.exists(path):
-        os.mkdir(path)
+def write_script(mod_name, path, class_nm, script_lst):
+    '''
+    Take a module name of the script type, path to output the script to, the definition class name and a list
+    of strings for the script and write it to disk
+    '''
 
-    f = open(os.path.join(path, 'ex_nexusformat_%s.py' % class_nm), 'w')
+    if not pathlib.Path(path).exists():
+        os.makedirs(path)
+    f = open(pathlib.PurePath(path, 'ex_%s_%s.py' % (mod_name,class_nm)), 'w')
     for l in script_lst:
         f.write(l + '\n')
     f.close()
 
 def build_class_db(class_dir='base_classes', desired_class=None, defdir=None, sym_args_dct={},report_symbols_only=False):
     '''
     build a nxdl definition into a dict
-        class_dir: either 'applications' or 'base_classes'
+        class_dir: one of the following: 'applications','base_classes' or 'contributed_definitions'
         desired_class: the name of a desired class definition such as 'NXstxm', if left as None then all class definitions\
                 will be returned.
         defdir: if the definitions are located somewhere other than in a subdir of nexpy
     '''
     if(defdir is None):
         class_path = pkg_resources.resource_filename('nexpy', 'definitions/%s' % class_dir)
     else:
-        class_path = os.path.join(defdir, class_dir)
+        class_path = pathlib.PurePath(defdir, class_dir)
 
-    nxdl_files = list(map(os.path.basename, glob.glob(os.path.join(class_path, '*.nxdl.xml'))))
+    nxdl_files = list(map(os.path.basename, glob.glob(str(pathlib.PurePath(class_path, '*.nxdl.xml')))))
     dct = {}
     if(desired_class):
-        nxdl_files = [os.path.join(class_path, '%s.nxdl.xml' % desired_class)]
+        nxdl_files = [pathlib.PurePath(class_path, '%s.nxdl.xml' % desired_class)]
 
     for nxdl_file in nxdl_files:
+        nxdl_file = str(nxdl_file)
         class_nm = nxdl_file.replace('.nxdl.xml', '')
-        if(class_nm.find(os.path.sep) > -1):
-            class_nm = class_nm.split(os.path.sep)[-1]
-        print('\nProcessing [%s]' % nxdl_file)
+        if(class_nm.find(pathlib.os.sep) > -1):
+            class_nm = class_nm.split(pathlib.os.sep)[-1]
+        print('\nProcessing [%s]' % nxdl_file.replace(pathlib.os.getcwd(),'.'))
         resp_dict, syms, docs = get_xml_paths(nxdl_file, sym_args_dct=sym_args_dct, report_symbols_only=report_symbols_only)
         dct[class_nm] = resp_dict
     return(dct, syms, docs)
@@ -1536,6 +1559,20 @@ def symbol_args_to_dict(arg_lst):
 
     return(dct)
 
+def process_nxdl(class_nm, def_subdir):
+    '''
+    given the class name and the destination directory, parse the nxdl file and produce an hdf5 file
+    as well as example scripts using h5py and nexusformat
+    '''
+    if class_nm.find('.nxdl.xml') > -1:
+        class_nm = class_nm.replace('.nxdl.xml', '')
+        build_class_db(def_subdir, desired_class=class_nm,
+                       defdir=def_dir, sym_args_dct=sym_args_dct,
+                       report_symbols_only=report_symbols_only)
+        dest_dir = pathlib.PurePath(pathlib.os.getcwd(), '..', 'autogenerated_examples', 'nxdl', def_subdir)
+        make_class_as_nf_file(class_nm, dest_dir, symbol_dct=sym_args_dct)
+
+
 if __name__ == '__main__':
     import argparse
 
@@ -1567,8 +1604,7 @@ def symbol_args_to_dict(arg_lst):
         print('\tProcess this entire directory [%s]' % args.directory)
         def_subdirs = [args.directory]
     else:
-        print('\tError: neither a specific definition or directory was specified so nothing to do')
-        exit()
+        print('Processing the definitions in the following sub directories', def_subdirs)
 
     if args.symbols:
         print('\tProcess using the following symbols [%s]' % args.symbols)
@@ -1579,11 +1615,11 @@ def symbol_args_to_dict(arg_lst):
         def_dir = args.nxdefdir
     else:
         #use the definitions in the installed nexpy
-        def_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'definitions')
+        def_dir = pathlib.PurePath(pathlib.os.getcwd(), '..', '..', 'definitions')
 
     #get the release version of the definitions
-    if os.path.exists(os.path.join(def_dir, 'NXDL_VERSION')):
-        f = open(os.path.join(def_dir, 'NXDL_VERSION'), 'r')
+    if pathlib.Path(pathlib.PurePath(def_dir, 'NXDL_VERSION')).exists():
+        f = open(pathlib.PurePath(def_dir, 'NXDL_VERSION'), 'r')
         l = f.readlines()
         f.close()
         rel_ver = l[0].replace('\n','')
@@ -1597,44 +1633,27 @@ def symbol_args_to_dict(arg_lst):
     #only search in applications and contributed_definitions subdirectories
     if(class_nm):
         for def_subdir in def_subdirs:
-            class_path = os.path.join(def_dir, def_subdir, class_nm + '.nxdl.xml')
-            if(os.path.exists(class_path)):
+            class_path = pathlib.PurePath(def_dir, def_subdir, class_nm + '.nxdl.xml')
+            if pathlib.Path(class_path).exists():
                 break
             else:
                 class_path = None
 
         if(class_path is None):
             print('Error: the class name [%s.nxdl.xml] doesnt exist in either of the applications or contributed_definitions subdirectories' % class_nm)
             exit()
+        else:
+            process_nxdl(class_path, def_subdir)
+            exit()
 
     files = None
     for def_subdir in def_subdirs:
-        files = sorted(os.listdir(os.path.join(def_dir, def_subdir)))
-        dest_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),'..', 'autogenerated_examples', 'nxdl', def_subdir)
-        # FOR TESTING #####################
-        #dest_dir = 'G:/github/nexusformat/exampledata/autogenerated_examples/nxdl/applications'
-        ##################################
-        if (class_nm is None):
-            do_exit = False
-        else:
-            do_exit = True
+        files = sorted(os.listdir(pathlib.PurePath(def_dir, def_subdir)))
         for class_path in files:
-            if class_path.find('.nxdl.xml') > -1:
-                if class_nm is None:
-                    class_nm = class_path.replace('.nxdl.xml','')
-
-                # path_dct, syms, docs = build_class_db(def_subdir, desired_class=class_nm,
-                #                                   defdir=def_dir, sym_args_dct=sym_args_dct,
-                #                                   report_symbols_only=report_symbols_only)
-                build_class_db(def_subdir, desired_class=class_nm,
-                                                        defdir=def_dir, sym_args_dct=sym_args_dct,
-                                                        report_symbols_only=report_symbols_only)
-                res = make_class_as_nf_file(class_nm, dest_dir, symbol_dct=sym_args_dct)
-
-                init_database()
-                class_nm = None
-                if do_exit:
-                    exit()
+            process_nxdl(class_path, def_subdir)
+            init_database()
+
+