From ad2c92520ce283235b4cef1895821c212010e6b4 Mon Sep 17 00:00:00 2001
From: caufieldjh <j.harry.caufield@gmail.com>
Date: Mon, 7 Mar 2022 14:37:31 -0500
Subject: [PATCH 1/4] Update to use kghub-downloader

---
 kg_covid_19/download.py | 2 +-
 setup.py                | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/kg_covid_19/download.py b/kg_covid_19/download.py
index 6a0170c2..0047e573 100644
--- a/kg_covid_19/download.py
+++ b/kg_covid_19/download.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 
-from .utils import download_from_yaml
+from kghub_downloader.download_utils import download_from_yaml
 
 
 def download(yaml_file: str, output_dir: str, ignore_cache: bool = False) -> None:
diff --git a/setup.py b/setup.py
index 835413ae..58bcd109 100644
--- a/setup.py
+++ b/setup.py
@@ -81,6 +81,7 @@ def find_version(*file_paths):
         'sphinx',
         'sphinx_rtd_theme',
         'recommonmark',
+        'kghub-downloader'
     ],
     extras_require=extras,
 )

From ee7b6c100a0ad7d3fec1345238d072013090f1a5 Mon Sep 17 00:00:00 2001
From: caufieldjh <j.harry.caufield@gmail.com>
Date: Mon, 7 Mar 2022 14:44:25 -0500
Subject: [PATCH 2/4] Fix mypy error

---
 kg_covid_19/download.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kg_covid_19/download.py b/kg_covid_19/download.py
index 0047e573..7fb10ab9 100644
--- a/kg_covid_19/download.py
+++ b/kg_covid_19/download.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 
-from kghub_downloader.download_utils import download_from_yaml
+from kghub_downloader.download_utils import download_from_yaml # type: ignore
 
 
 def download(yaml_file: str, output_dir: str, ignore_cache: bool = False) -> None:

From 35755d76576dfbb57b742beb07778bae85d3ef66 Mon Sep 17 00:00:00 2001
From: caufieldjh <j.harry.caufield@gmail.com>
Date: Tue, 15 Mar 2022 15:07:25 -0400
Subject: [PATCH 3/4] Update reqs and Jenkins for multi-indexer

---
 Jenkinsfile | 4 ++--
 setup.py    | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 56f43b2a..b633042d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -191,7 +191,7 @@ pipeline {
                                 //
                                 // put $S3PROJECTDIR/$BUILDSTARTDATE/ and $S3PROJECTDIR/current in s3 bucket
                                 //
-                                sh '. venv/bin/activate && python3.8 ./go-site/scripts/directory_indexer.py -v --inject ./go-site/scripts/directory-index-template.html --directory $S3PROJECTDIR --prefix https://kg-hub.berkeleybop.io/$S3PROJECTDIR/ -x -u'
+                                sh '. venv/bin/activate && python3.8 multi_indexer -v --directory $S3PROJECTDIR --prefix https://kg-hub.berkeleybop.io/$S3PROJECTDIR/ -x -u'
                                 // for existing builds on s3, we just made an index.html that will clobber the existing (correct) s3 index.html
                                 // here we download the existing index.html and clobber the local one instead
                                 sh ". venv/bin/activate && for dir in `s3cmd ls s3://kg-hub-public-data/kg-covid-19/ | grep '\\/\$' | awk '{print \$NF}' | grep -w -v -E 'raw|current' | xargs -n1 basename`; do s3cmd get --force --continue s3://kg-hub-public-data/kg-covid-19/\$dir/index.html $S3PROJECTDIR/\$dir/ || true; done"
@@ -200,7 +200,7 @@ pipeline {
 
                                 // Build the top level index.html
                                 // "External" packages required to run these scripts.
-                                sh '. venv/bin/activate && python3.8 ./go-site/scripts/bucket-indexer.py --credentials $AWS_JSON --bucket kg-hub-public-data --inject ./go-site/scripts/directory-index-template.html --prefix https://kg-hub.berkeleybop.io/ > top-level-index.html'
+                                sh '. venv/bin/activate && python3.8 multi_indexer --bucket kg-hub-public-data --prefix https://kg-hub.berkeleybop.io/ > top-level-index.html'
                                 sh '. venv/bin/activate && s3cmd -c $S3CMD_CFG put --acl-public --mime-type=text/html --cf-invalidate top-level-index.html s3://kg-hub-public-data/index.html'
 
                                 // Invalidate the CDN now that the new files are up.
diff --git a/setup.py b/setup.py
index 58bcd109..f01c217b 100644
--- a/setup.py
+++ b/setup.py
@@ -81,7 +81,8 @@ def find_version(*file_paths):
         'sphinx',
         'sphinx_rtd_theme',
         'recommonmark',
-        'kghub-downloader'
+        'kghub-downloader',
+        'multi-indexer'
     ],
     extras_require=extras,
 )

From 0776a974cb4e165e81dffe8ab241348f481dcc11 Mon Sep 17 00:00:00 2001
From: caufieldjh <j.harry.caufield@gmail.com>
Date: Tue, 15 Mar 2022 15:14:35 -0400
Subject: [PATCH 4/4] Pin elasticsearch to 7.17.1 to avoid breaking

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index f01c217b..caef7b7e 100644
--- a/setup.py
+++ b/setup.py
@@ -77,7 +77,7 @@ def find_version(*file_paths):
         'pyyaml',
         'bmt',
         'ensmallen_graph==0.4.3',
-        'elasticsearch',
+        'elasticsearch==7.17.1',
         'sphinx',
         'sphinx_rtd_theme',
         'recommonmark',