Add config file, options

.github/workflows/manual-build.yml

@@ -0,0 +1,11 @@
+---
+name: Manual Build & Push
+on:
+ workflow_dispatch:
+jobs:
+  build-push:
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}-develop'
+      tags: br-${{ github.ref_name }}
+    secrets: inherit

.github/workflows/pr_build.yml

@@ -0,0 +1,43 @@
+---
+name: Pull Request Build, Tag, & Push
+on:
+  pull_request:
+    branches:
+      - develop
+      - main
+      - master
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - closed
+jobs:
+  build-develop-open:
+    if: github.base_ref == 'develop' && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_build.yml@main
+    secrets: inherit
+  build-develop-merge:
+    if: github.base_ref == 'develop' && github.event.pull_request.merged == true
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}-develop'
+      tags: pr-${{ github.event.number }},latest
+    secrets: inherit
+  build-main-open:
+    if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}'
+      tags: pr-${{ github.event.number }}
+    secrets: inherit
+  build-main-merge:
+    if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}'
+      tags: pr-${{ github.event.number }},latest-rc
+    secrets: inherit
+  trivy-scans:
+    if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main
+    secrets: inherit

.github/workflows/release-main.yml

@@ -0,0 +1,25 @@
+---
+name: Release - Build & Push Image
+on:
+  release:
+    branches:
+      - main
+      - master
+    types: [ published ]
+jobs:
+  check-source-branch:
+    uses: kbase/.github/.github/workflows/reusable_validate-branch.yml@main
+    with:
+      build_branch: '${{ github.event.release.target_commitish }}'
+  validate-release-tag:
+    needs: check-source-branch
+    uses: kbase/.github/.github/workflows/reusable_validate-release-tag.yml@main
+    with:
+      release_tag: '${{ github.event.release.tag_name }}'
+  build-push:
+    needs: validate-release-tag
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}'
+      tags: '${{ github.event.release.tag_name }},latest'
+    secrets: inherit

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.9.16
+RUN apt-get update
+RUN apt-get install -y vim
+
+RUN curl https://dl.min.io/client/mc/release/linux-amd64/mc \
+  --create-dirs \
+  -o $HOME/minio-binaries/mc
+
+RUN chmod +x $HOME/minio-binaries/mc
+RUN mv $HOME/minio-binaries/mc /bin/mc
+
+RUN pip install boto3 pymongo
+RUN mkdir /s3transfer
+COPY *.py /s3transfer/
+COPY *.ini /s3transfer/
+WORKDIR /s3transfer
+ENTRYPOINT ["/bin/bash", "-c"]
+CMD "/bin/sleep"

README.md

@@ -1,3 +1,16 @@
 # Shock to S3 scripts
 
-Scripts for moving / transforming KBase data files and database records as part of the Shock to S3 data storage conversion.
+Scripts for moving and transforming KBase data files and database records as part of the Shock to S3 data storage conversion.
+
+# Transferring S3 to S3
+* Use syncs3tos3.py
+* fill out syncs3blobstore.ini example config file
+* you may need to edit /etc/hosts for mongo replicas to correctly get queried e.g
+
+```
+X.X.1.1 next01
+X.X.1.6 next06
+```
+* for the first run, it will create a "now" file with the timestamp. You can cancel the run, and edit the file, and set the date to the year 2000
+* you can monitor progress in the rsync log at blobstoresync.log 
+* it may help to enable debug mode or add print statements to help figure out what the script is doing

attic/shockMongoRecordsToS3.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+
+'''
+This script converts a subset of Shock (https://github.com/kbase/Shock) node records to
+BlobStore (https://github.com/kbase/blobstore) records in MongoDB only.
+The script does not alter the Shock backend records and may be re-run multiple times without issue.
+It also does not copy any files to S3.
+
+The script expects key names based on a uuid in the same way Shock directories are
+based on the UUID; for example the UUID 06f5d3ec-8ebf-4d32-8c1c-41e27e40b7fd would be
+
+06/f5/d3/06f5d3ec-8ebf-4d32-8c1c-41e27e40b7fd
+
+... as a key in S3 or a directory tree in Shock.
+
+To use:
+1) Start the blobstore at least once so that the proper indexes are created in MongoDB.
+2) Fill in the configuration variables below and run the script normally.
+'''
+
+import configparser
+import argparse
+import datetime
+import uuid
+from pprint import pprint
+from pymongo.mongo_client import MongoClient
+from pymongo import UpdateOne
+from pymongo.errors import ServerSelectionTimeoutError
+from pymongo.errors import BulkWriteError
+
+parser = argparse.ArgumentParser(description='Import Shock Mongo data to blobstore Mongo.')
+parser.add_argument('--config-file', dest='configfile', required=True,
+		    help='Path to config file (INI format). (required)')
+args = parser.parse_args()
+
+configfile=args.configfile
+conf=configparser.ConfigParser()
+conf.read(configfile)
+
+###### CONFIGURATION VARIABLES ######
+
+CONFIG_MONGO_SHOCK_HOST = conf['shock']['mongo_host']
+CONFIG_MONGO_SHOCK_DATABASE = conf['shock']['mongo_database']
+CONFIG_MONGO_SHOCK_USER = conf['shock']['mongo_user']
+CONFIG_MONGO_SHOCK_PWD = conf['shock']['mongo_pwd']
+
+CONFIG_MONGO_BLOBSTORE_HOST = conf['blobstore']['mongo_host']
+CONFIG_MONGO_BLOBSTORE_DATABASE = conf['blobstore']['mongo_database']
+CONFIG_MONGO_BLOBSTORE_USER = conf['blobstore']['mongo_user']
+CONFIG_MONGO_BLOBSTORE_PWD = conf['blobstore']['mongo_pwd']
+
+CONFIG_SHOCK_WS_UUID = conf['shock']['ws_uuid']
+
+# dumb but lazy
+CONFIG_START_YEAR = conf['shock']['start_year'] or 2000
+CONFIG_START_MONTH = conf['shock']['start_month'] or 1
+CONFIG_START_DAY = conf['shock']['start_day'] or 1
+CONFIG_END_YEAR = conf['shock']['end_year'] or 2037
+CONFIG_END_MONTH = conf['shock']['end_month'] or 12
+CONFIG_END_DAY = conf['shock']['end_day'] or 28
+
+CONFIG_START_DATE = datetime.datetime(int(CONFIG_START_YEAR),int(CONFIG_START_MONTH),int(CONFIG_START_DAY),0,0,0)
+CONFIG_END_DATE = datetime.datetime(int(CONFIG_END_YEAR),int(CONFIG_END_MONTH),int(CONFIG_END_DAY),0,0,0)
+
+CONFIG_BATCH_SIZE = 10000
+
+#### END CONFIGURATION VARIABLES ####
+
+BS_COL_NODES = 'nodes'
+BS_COL_USERS = 'users'
+
+BS_KEY_USERS_ID = 'id'
+BS_KEY_USERS_USER = 'user'
+
+BS_KEY_NODES_ID = 'id'
+BS_KEY_NODES_OWNER = 'own'
+BS_KEY_NODES_READERS = 'read'
+BS_KEY_NODES_FILENAME = 'fname'
+BS_KEY_NODES_FORMAT = 'fmt'
+BS_KEY_NODES_SIZE = 'size'
+BS_KEY_NODES_MD5 = 'md5'
+BS_KEY_NODES_STORED = 'time'
+BS_KEY_NODES_PUBLIC = 'pub'
+
+
+SHOCK_COL_NODES = 'Nodes'
+SHOCK_COL_USERS = 'Users'
+
+SHOCK_KEY_USERS_ID = 'uuid'
+SHOCK_KEY_USERS_USER = 'username'
+
+SHOCK_KEY_NODES_ID = 'id'
+SHOCK_KEY_NODES_CREATED = 'created_on'
+SHOCK_KEY_NODES_FILE = 'file'
+SHOCK_KEY_NODES_FILE_NAME = 'name'
+SHOCK_KEY_NODES_FILE_SIZE = 'size'
+SHOCK_KEY_NODES_FILE_CHKSUM = 'checksum'
+SHOCK_KEY_NODES_FILE_CHKSUM_MD5 = 'md5'
+SHOCK_KEY_NODES_FILE_FORMAT = 'format'
+SHOCK_KEY_NODES_ACLS = 'acl'
+SHOCK_KEY_NODES_ACLS_OWNER = 'owner'
+SHOCK_KEY_NODES_ACLS_READERS = 'read'
+SHOCK_KEY_NODES_ACLS_PUBLIC = 'public'
+
+def main():
+    shockdb = get_mongo_client(CONFIG_MONGO_SHOCK_HOST, CONFIG_MONGO_SHOCK_DATABASE,
+        CONFIG_MONGO_SHOCK_USER, CONFIG_MONGO_SHOCK_PWD)[CONFIG_MONGO_SHOCK_DATABASE]
+    bsdb = get_mongo_client(CONFIG_MONGO_BLOBSTORE_HOST, CONFIG_MONGO_BLOBSTORE_DATABASE,
+        CONFIG_MONGO_BLOBSTORE_USER, CONFIG_MONGO_BLOBSTORE_PWD)[CONFIG_MONGO_BLOBSTORE_DATABASE]
+
+    query = { 'acl.owner': { '$ne': CONFIG_SHOCK_WS_UUID }, 'created_on': { '$gt': CONFIG_START_DATE, '$lt': CONFIG_END_DATE } }
+    #    print(query)
+
+    totalNodes = shockdb[SHOCK_COL_NODES].count_documents(query)
+
+    count = 0
+    seenusers = {}
+    doc_update_list = []
+    lastPrint = 'Processed {}/{} records'.format(count,totalNodes)
+    print (lastPrint)
+
+    for node in shockdb[SHOCK_COL_NODES].find(query,batch_size=CONFIG_BATCH_SIZE,no_cursor_timeout=True):
+#        print(node['id'])
+        #print (node['id'][0:2] + '/' + node['id'][2:4] + '/' + node['id'][4:6] + '/' + node['id'] + '/' + node['id'] + '.data')
+
+	bsnode = toBSNode(node, seenusers, shockdb, bsdb)
+#	pprint(bsnode)
+        doc_update_list.append(UpdateOne(
+            { BS_KEY_NODES_ID: node['id'] } ,
+	    { '$set': bsnode }   
+		 , upsert=True
+	))
+
+#        bsdb[BS_COL_NODES].update_one({BS_KEY_NODES_ID: node}, {'$set': bsnode}, upsert=True)
+        count += 1
+
+	if len(doc_update_list) % CONFIG_BATCH_SIZE == 0:
+	    bulk_update_blobstore_nodes(bsdb, doc_update_list)
+            doc_update_list = []
+
+	if count % CONFIG_BATCH_SIZE == 0:
+            lastPrint = 'Processed {}/{} records'.format(count,totalNodes)
+            print (lastPrint)
+
+# last update
+    if len(doc_update_list) > 0:
+        bulk_update_blobstore_nodes(bsdb, doc_update_list)
+        lastPrint = 'Processed {}/{} records'.format(count,totalNodes)
+        print(lastPrint)
+
+def toBSNode(shocknode, seenusers, shockdb, bsdb):
+    n = shocknode
+#    pprint(n['id'])
+    owner = n[SHOCK_KEY_NODES_ACLS][SHOCK_KEY_NODES_ACLS_OWNER]
+    try:
+        md5 = n[SHOCK_KEY_NODES_FILE][SHOCK_KEY_NODES_FILE_CHKSUM][SHOCK_KEY_NODES_FILE_CHKSUM_MD5]
+    except KeyError:
+	pprint('node ' + n['id'] + ' has no md5, assigning None')
+	md5 = None
+    readers = n[SHOCK_KEY_NODES_ACLS][SHOCK_KEY_NODES_ACLS_READERS]
+    pub = SHOCK_KEY_NODES_ACLS_PUBLIC in readers
+    while SHOCK_KEY_NODES_ACLS_PUBLIC in readers: readers.remove(SHOCK_KEY_NODES_ACLS_PUBLIC)  
+
+    bsnode = {
+        BS_KEY_NODES_ID: n[SHOCK_KEY_NODES_ID],
+        BS_KEY_NODES_OWNER: get_user(owner, seenusers, shockdb, bsdb),
+        BS_KEY_NODES_READERS: [get_user(r, seenusers, shockdb, bsdb) for r in readers],
+        BS_KEY_NODES_STORED: n[SHOCK_KEY_NODES_CREATED],
+        BS_KEY_NODES_FILENAME: n[SHOCK_KEY_NODES_FILE][SHOCK_KEY_NODES_FILE_NAME],
+        BS_KEY_NODES_SIZE: n[SHOCK_KEY_NODES_FILE][SHOCK_KEY_NODES_FILE_SIZE],
+        BS_KEY_NODES_FORMAT: n[SHOCK_KEY_NODES_FILE][SHOCK_KEY_NODES_FILE_FORMAT],
+        BS_KEY_NODES_MD5: md5,
+        BS_KEY_NODES_PUBLIC: pub,
+    }
+    return bsnode
+
+def get_user(uuid, seenusers, shockdb, bsdb):
+    if uuid in seenusers:
+        return {BS_KEY_USERS_ID: uuid, BS_KEY_USERS_USER: seenusers[uuid]}
+
+    u = shockdb[SHOCK_COL_USERS].find_one({SHOCK_KEY_USERS_ID: uuid})
+    if not u:
+        raise ValueError('Missing user in Shock ' + uuid)
+    bsdb[BS_COL_USERS].update_one(
+        {BS_KEY_USERS_ID: uuid},
+        {'$set': {BS_KEY_USERS_USER: u[SHOCK_KEY_USERS_USER]}},
+        upsert=True)
+    seenusers[uuid] = u[SHOCK_KEY_USERS_USER]
+    return {BS_KEY_USERS_ID: uuid, BS_KEY_USERS_USER: u[SHOCK_KEY_USERS_USER]}
+
+def toUUID(s3key):
+    u = s3key.split('/')
+    uuidStr = u[3]
+    if uuidStr[0:2] != u[0] or uuidStr[2:4] != u[1] or uuidStr[4:6] != u[2]:
+        raise ValueError("Illegal S3 key: " + uuidStr)
+    try:
+        uuid.UUID(hex=uuidStr)
+    except TypeError as _:
+        raise ValueError("Illegal S3 key: " + uuidStr)
+    return uuidStr
+
+def get_mongo_client(host, db, user, pwd):
+    if user:
+        return MongoClient(host, authSource=db, username=user, password=pwd, retryWrites=False)
+    else:
+        return MongoClient(host, retryWrites=False)
+
+
+def bulk_update_blobstore_nodes(db, doc_update_list):
+    try:
+        update_result = db[BS_COL_NODES].bulk_write(doc_update_list,ordered=False)
+    except BulkWriteError as bwe:
+        print(bwe.details)
+    print('nInserted: ' + str(update_result.bulk_api_result['nInserted']) + ' ; nUpserted: ' + str(update_result.bulk_api_result['nUpserted']) + ' ; writeErrors: ' + str(update_result.bulk_api_result['writeErrors']))
+#    pprint(update_result.bulk_api_result)
+
+if __name__ == '__main__':
+    main()
+