Changing packaging structure

.github/workflows/create-gfdl-catalog.yml

@@ -29,10 +29,10 @@ jobs:
       run: python tests/make_sample_data.py
     - name: 'Generate catalog'
       run: |
-        $CONDA/envs/catalogbuilder/bin/python scripts/gen_intake_gfdl.py archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp gfdl_autotest
+        $CONDA/envs/catalogbuilder/bin/python catalogbuilder/scripts/gen_intake_gfdl.py archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp gfdl_autotest
     - name: 'Generate catalog with yaml'
       run: |
-         $CONDA/envs/catalogbuilder/bin/python scripts/gen_intake_gfdl.py --config tests/test_config.yaml
+         $CONDA/envs/catalogbuilder/bin/python catalogbuilder/scripts/gen_intake_gfdl.py --config tests/test_config.yaml
     - name: upload-artifacts1
       uses: actions/upload-artifact@v4
       with:
@@ -50,5 +50,5 @@ jobs:
         $CONDA/envs/catalogbuilder/bin/pytest -v --runxfail
     - name: Test for completeness
       run: | 
-        $CONDA/envs/catalogbuilder/bin/python scripts/test_catalog.py -tf gfdl_autotest.json cats/gfdl_template.json
-        $CONDA/envs/catalogbuilder/bin/python scripts/test_catalog.py -tf cats/gfdl_autotest_from_yaml.json 
+        $CONDA/envs/catalogbuilder/bin/python catalogbuilder/scripts/test_catalog.py -tf gfdl_autotest.json catalogbuilder/cats/gfdl_template.json
+        $CONDA/envs/catalogbuilder/bin/python catalogbuilder/scripts/test_catalog.py -tf catalogbuilder/cats/gfdl_autotest_from_yaml.json 

.github/workflows/publish-conda.yml

@@ -1,7 +1,8 @@
 name: publish_conda
 on:
-  push:
+  pull_request:
     branches:
+    #for testing the restructuring, back on push only later.
     - main
 jobs:
   build_and_publish:

catalogbuilder/__init__.py

@@ -0,0 +1 @@
+

scripts/gen_intake_gfdl_notebook.ipynb → catalogbuilder/scripts/gen_intake_gfdl_notebook.ipynb

@@ -56,7 +56,7 @@
     }
    ],
    "source": [
-    "from scripts import gen_intake_gfdl\n",
+    "from catalogbuilder.scripts import gen_intake_gfdl\n",
     "import sys,os\n",
     "\n",
     "######USER input begins########\n",

scripts/gen_intake_gfdl_runner.py → catalogbuilder/scripts/gen_intake_gfdl_runner.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
-from scripts import gen_intake_gfdl
+#TODO test after conda pkg is published and make changes as needed 
+from catalogbuilder.scripts import gen_intake_gfdl
 import sys
 
 input_path = "/archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp/"

scripts/gen_intake_gfdl_runner_config.py → catalogbuilder/scripts/gen_intake_gfdl_runner_config.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from scripts import gen_intake_gfdl
+from catalogbuilder.scripts import gen_intake_gfdl
 import sys
 
 sys.argv = ['input_path','--config', '/home/a1r/github/CatalogBuilder/scripts/configs/config-example.yml']

scripts/test_catalog.py → catalogbuilder/scripts/test_catalog.py

@@ -23,7 +23,7 @@ def main(json_path,json_template_path,test_failure):
     if json_template_path:
         json_template = json.load(open(json_template_path))
     else:
-        json_template = json.load(open('cats/gfdl_template.json'))
+        json_template = json.load(open('catalogbuilder/cats/gfdl_template.json'))
 
     #Validate JSON against JSON template
     comp = (diff(j,json_template))

meta.yaml

@@ -1,9 +1,10 @@
 package:
-  name: intakebuilder
+  name: catalogbuilder
   version: 2.0.1
 
 source:
-  git_url: https://github.com/aradhakrishnanGFDL/CatalogBuilder.git
+  git_url: https://github.com/NOAA-GFDL/CatalogBuilder.git
+  git_branch: subpkgtest
   path: .
 
 build:
@@ -16,16 +17,17 @@ requirements:
     - pip 
   run:
     - python
+    - pytest
     - click
-    - pandas
     - xarray
-    - pytest
-    - conda-forge::intake-esm=2023.7.7
+    - pandas
     - jsondiff
+    - conda-forge::intake-esm=2023.7.7
+    - boto3
 test:
   imports:
-    - intakebuilder 
-    - scripts.gen_intake_gfdl
-
+    - catalogbuilder
+    - catalogbuilder.scripts
+    - catalogbuilder.cats
 about:
   home:

setup.py

@@ -14,8 +14,9 @@
         'click',
         'xarray',
         'pandas',
-        'boto3',
-        'botocore'
+        'jsondiff',
+        'intake-esm',
+        'boto3'
     ]
 )
 

tests/test_config.yaml

@@ -38,4 +38,4 @@ output_file_template: ['modeling_realm','temporal_subset','variable_id']
 #######################################################
 
 input_path: "archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp/" #"ENTER INPUT PATH HERE" #Example: /Users/ar46/archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp/"
-output_path: "cats/gfdl_autotest_from_yaml" # ENTER NAME OF THE CSV AND JSON, THE SUFFIX ALONE. e.g catalog (the builder then generates catalog.csv and catalog.json. This can also be an absolute path)
+output_path: "catalogbuilder/cats/gfdl_autotest_from_yaml" # ENTER NAME OF THE CSV AND JSON, THE SUFFIX ALONE. e.g catalog (the builder then generates catalog.csv and catalog.json. This can also be an absolute path)

tests/test_import.py

@@ -1,10 +1,21 @@
+import os,sys
 def check_import():
  try:
-  from intakebuilder import getinfo, localcrawler, CSVwriter
-  print("Imported intakebuilder and getinfo, localcrawler, CSVwriter ")
+   from intakebuilder import gfdlcrawler, CSVwriter, configparser
  except ImportError:
-  raise ImportError('Error importing intakebuilder and other packages')
-  return -97
+    print("The module intakebuilder is not installed. Do you have intakebuilder in your sys.path or have you activated the conda environment with the intakebuilder package in it? ")
+    print("Attempting again with adjusted sys.path ")
+    try:
+       #candobetter with actual conda package built before this, but this is a decent test for those that do not use conda package
+       sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+"/catalogbuilder")
+       from intakebuilder import gfdlcrawler, CSVwriter, builderconfig, configparser
+    except:
+       print("Unable to adjust sys.path")
+       raise ImportError('Error importing intakebuilder and other packages')
+       return -97
  return True
+
 def test_import():
     assert check_import() == True
+
+

tests/test_ingestion.py

@@ -21,7 +21,7 @@ def load_cat(catspec=None):
   return cat
 def test_loadcat():
   #TODO generate csv on the fly, check if its readable etc
-  catspec = pathlib.Path(os.path.dirname(__file__)).parent / 'cats/gfdl_template.json'
+  catspec = pathlib.Path(os.path.dirname(__file__)).parent / 'catalogbuilder/cats/gfdl_template.json'
   #TODO generate test catalog on the fly, push spec to the test directory
   cat = load_cat((str(catspec)))
   try: