Merge pull request #95 from refgenie/dev

v0.9.0
refgenie · Jul 1, 2020 · a186c54 · a186c54
2 parents 8c9903e + 514db67
commit a186c54
Show file tree

Hide file tree

Showing 14 changed files with 93 additions and 61 deletions.
diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml
@@ -0,0 +1,41 @@
+name: Run pytests
+
+on:
+  push:
+    branches: [master, dev]
+  pull_request:
+    branches: [master, dev]
+
+jobs:
+  pytest:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+        os: [ubuntu-latest, macos-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dev dependancies
+      run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi
+
+    - name: Install test dependancies
+      run: if [ -f requirements/requirements-test.txt ]; then pip install -r requirements/requirements-test.txt; fi
+
+    - name: Install package
+      run: python -m pip install .
+
+    - name: Run pytest tests
+      run: pytest tests --remote-data --cov=./ --cov-report=xml
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      with:
+        file: ./coverage.xml
+        name: py-${{ matrix.python-version }}-${{ matrix.os }}
diff --git a/.travis.yml b/.travis.yml
diff --git a/README.md b/README.md
@@ -1,9 +1,9 @@
 # refgenconf
 
-[![Build Status](https://travis-ci.org/databio/refgenconf.svg?branch=master)](https://travis-ci.org/databio/refgenconf)
-[![Coverage Status](https://coveralls.io/repos/github/refgenie/refgenconf/badge.svg?branch=master)](https://coveralls.io/github/refgenie/refgenconf?branch=master)
+![Run pytests](https://github.com/refgenie/refgenconf/workflows/Run%20pytests/badge.svg)
+[![codecov](https://codecov.io/gh/refgenie/refgenconf/branch/master/graph/badge.svg)](https://codecov.io/gh/refgenie/refgenconf)
 [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg?style=flat)](http://bioconda.github.io/recipes/refgenconf/README.html)
 
-Configuration object for refgenie *et al.*
+Configuration object for [refgenie](https://doi.org/10.1093/gigascience/giz149) *et al.*
 
 Documentation for `refgenconf` can be found with the [primary documentation for refgenie](http://refgenie.databio.org).
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -2,6 +2,12 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. 
 
+## [0.9.0] - 2020-07-01
+
+### Changed
+- `pull` so it does not remove asset after overwrite decision, wait for the archive download to finish
+- file locking mechanism enhancements
+
 ## [0.8.0] - 2020-06-25
 
 ### Added

diff --git a/refgenconf/_version.py b/refgenconf/_version.py
@@ -1 +1 @@
-__version__ = "0.8.0"
+__version__ = "0.9.0"
diff --git a/refgenconf/refgenconf.py b/refgenconf/refgenconf.py
@@ -49,15 +49,19 @@ def handle(sig, frame):
 class RefGenConf(yacman.YacAttMap):
     """ A sort of oracle of available reference genome assembly assets """
 
-    def __init__(self, filepath=None, entries=None, writable=False, wait_max=60):
+    def __init__(self, filepath=None, entries=None, writable=False, wait_max=60,
+                 skip_read_lock=False):
         """
         Create the config instance by with a filepath or key-value pairs.
 
         :param str filepath: a path to the YAML file to read
         :param Iterable[(str, object)] | Mapping[str, object] entries:
             config filepath or collection of key-value pairs
         :param bool writable: whether to create the object with write capabilities
-        :param int wait_max: how long to wait for creating an object when the file that data will be read from is locked
+        :param int wait_max: how long to wait for creating an object when the
+            file that data will be read from is locked
+        :param bool skip_read_lock: whether the file should not be locked for
+            reading when object is created in read only mode
         :raise refgenconf.MissingConfigDataError: if a required configuration
             item is missing
         :raise ValueError: if entries is given as a string and is not a file
@@ -66,7 +70,9 @@ def __init__(self, filepath=None, entries=None, writable=False, wait_max=60):
         def _missing_key_msg(key, value):
             _LOGGER.debug("Config lacks '{}' key. Setting to: {}".format(key, value))
 
-        super(RefGenConf, self).__init__(filepath=filepath, entries=entries, writable=writable, wait_max=wait_max)
+        super(RefGenConf, self).__init__(filepath=filepath, entries=entries,
+                                         writable=writable, wait_max=wait_max,
+                                         skip_read_lock=skip_read_lock)
         genomes = self.setdefault(CFG_GENOMES_KEY, PXAM())
         if not isinstance(genomes, PXAM):
             if genomes:
@@ -639,7 +645,7 @@ def _raise_unpack_error():
                 no_asset_json.append(server_url)
                 if num_servers == len(self[CFG_SERVERS_KEY]):
                     _LOGGER.error("Asset '{}/{}:{}' not available on any of the following servers: {}".
-                                  format(genome, asset, determined_tag, ", ".join(no_asset_json)))
+                                  format(genome, asset, determined_tag, ", ".join(self[CFG_SERVERS_KEY])))
                     return _null_return()
                 continue
 
@@ -655,21 +661,17 @@ def _raise_unpack_error():
             # check if the genome/asset:tag exists and get request user decision
             if os.path.exists(tag_dir):
                 def preserve():
-                    _LOGGER.debug("Preserving existing: {}".format(tag_dir))
+                    _LOGGER.info("Preserving existing: {}".format(tag_dir))
                     return _null_return()
-
-                def msg_overwrite():
-                    _LOGGER.debug("Overwriting: {}".format(tag_dir))
-                    shutil.rmtree(tag_dir)
                 if force is False:
                     return preserve()
                 elif force is None:
                     if not query_yes_no("Replace existing ({})?".format(tag_dir), "no"):
                         return preserve()
                     else:
-                        msg_overwrite()
+                        _LOGGER.debug("Overwriting: {}".format(tag_dir))
                 else:
-                    msg_overwrite()
+                    _LOGGER.debug("Overwriting: {}".format(tag_dir))
 
             # check asset digests local-server match for each parent
             [self._chk_digest_if_avail(genome, x, server_url)
@@ -717,7 +719,8 @@ def msg_overwrite():
             new_checksum = checksum(filepath)
             old_checksum = archive_data and archive_data.get(CFG_ARCHIVE_CHECKSUM_KEY)
             if old_checksum and new_checksum != old_checksum:
-                _LOGGER.error("Checksum mismatch: ({}, {})".format(new_checksum, old_checksum))
+                _LOGGER.error("Downloaded archive ('{}') checksum mismatch: ({}, {})".
+                              format(filepath, new_checksum, old_checksum))
                 return _null_return()
             else:
                 _LOGGER.debug("Matched checksum: '{}'".format(old_checksum))
@@ -729,6 +732,9 @@ def msg_overwrite():
                     # directory with the asset data inside and we transfer it
                     # to the tag-named subdirectory
                     untar(filepath, tmpdir)
+                    if os.path.isdir(tag_dir):
+                        shutil.rmtree(tag_dir)
+                        _LOGGER.info("Removed existing directory: {}".format(tag_dir))
                     shutil.move(os.path.join(tmpdir, asset), tag_dir)
                 if os.path.isfile(filepath):
                     os.remove(filepath)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
@@ -2,6 +2,5 @@ attmap>=0.12.5
 pyyaml
 requests
 tqdm>=4.38.0
-ubiquerg>=0.6.0
-yacman>=0.6.8
+yacman>=0.6.9
 future
diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt
@@ -1,5 +0,0 @@
-pytest>=3.0.7
-pytest-remotedata
-pyyaml>=5
-ubiquerg>=0.3
-veracitools

diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt
@@ -1,4 +1,6 @@
-coveralls>=1.1
-pytest-cov==2.6.1
+pytest-cov
+pytest
 pytest-remotedata
+mock
+veracitools
 git+git://github.com/databio/refgenie_myplugin@master#egg=refgenie_myplugin
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -127,7 +127,7 @@ def made_genome_config_file(temp_genome_config_file):
     """ Make the test session's genome config file. """
     genome_folder = os.path.dirname(temp_genome_config_file)
     extra_kv_lines = ["{}: {}".format(CFG_FOLDER_KEY, genome_folder),
-                      "{}: {}".format(CFG_SERVERS_KEY, "http://staging.refgenomes.databio.org/"),
+                      "{}: {}".format(CFG_SERVERS_KEY, "https://refgenomes.databio.org/"),
                       "{}: {}".format(CFG_VERSION_KEY, package_version),
                       "{}:".format(CFG_GENOMES_KEY)]
     gen_data_lines = PathExAttMap(CONF_DATA).get_yaml_lines()

diff --git a/tests/data/genomes.yaml b/tests/data/genomes.yaml
@@ -2,19 +2,6 @@ config_version: 0.3
 genome_folder: /tmp
 genome_servers: ['http://refgenomes.databio.org']
 genomes:
-  human_repeats:
-    assets:
-      fasta:
-        tags:
-          default:
-            seek_keys:
-              fasta: human_repeats.fa
-              fai: human_repeats.fa.fai
-              chrom_sizes: human_repeats.chrom.sizes
-            asset_parents: []
-            asset_path: fasta
-            asset_digest: 4a749d4e74b057d0efa0c8398ebcb871
-        default_tag: default
   mouse_chrM2x:
     assets:
       bwa_index:
@@ -48,3 +35,16 @@ genomes:
             asset_path: fasta
             asset_digest: 4eb430296bc02ed7e4006624f1d5ac53
         default_tag: default
+  human_repeats:
+    assets:
+      fasta:
+        tags:
+          default:
+            seek_keys:
+              fasta: human_repeats.fa
+              fai: human_repeats.fa.fai
+              chrom_sizes: human_repeats.chrom.sizes
+            asset_parents: []
+            asset_path: fasta
+            asset_digest: 4a749d4e74b057d0efa0c8398ebcb871
+        default_tag: default
diff --git a/tests/data/genomes_v2.yaml b/tests/data/genomes_v2.yaml
@@ -1,6 +1,6 @@
 config_version: 0.2
 genome_folder: /tmp
-genome_server: http://staging.refgenomes.databio.org
+genome_server: https://refgenomes.databio.org/
 genomes:
   rCRSd:
     assets:

diff --git a/tests/test_1pull_asset.py b/tests/test_1pull_asset.py
@@ -120,6 +120,8 @@ def test_pull_asset_updates_genome_config(cfg_file, gname, aname, tname):
 def test_pull_asset_works_with_nonwritable_and_writable_rgc(cfg_file, gname, aname, tname, state):
     rgc = RefGenConf(filepath=cfg_file, writable=state)
     remove_asset_and_file(rgc, gname, aname, tname)
+    print("\nPulling; genome: {}, asset: {}, tag: {}\n".format(gname, aname, tname))
     with mock.patch("refgenconf.refgenconf.query_yes_no", return_value=True):
-        print("\nPulling; genome: {}, asset: {}, tag: {}\n".format(gname, aname, tname))
         rgc.pull(gname, aname, tname)
+    if state:
+        rgc.make_readonly()
diff --git a/tests/test_list_remote.py b/tests/test_list_remote.py
@@ -13,7 +13,7 @@ def test_list_remote(rgc, tmpdir):
     new_rgc = RefGenConf(entries={CFG_FOLDER_KEY: tmpdir.strpath,
                           CFG_SERVERS_KEY: DEFAULT_SERVER,
                           CFG_GENOMES_KEY: rgc[CFG_GENOMES_KEY]})
-    new_rgc[CFG_SERVERS_KEY] = "http://staging.refgenomes.databio.org"
+    new_rgc[CFG_SERVERS_KEY] = "https://refgenomes.databio.org/"
     print("NEW RGC KEYS: {}".format(list(new_rgc.keys())))
     with mock.patch("refgenconf.refgenconf._read_remote_data",
                     return_value=rgc.genomes):