Merge BugSwarm/client into monorepo (#362)

* Add bugswarm/client; edit `setup.py`s

* Add bugswarm-client to build-artifacts action

* Refactor build-artifacts.yml

* Publish both common and client to PyPI

* Manually update client version pending discussion

* Update bugswarm-client alongside bugswarm-common

* Use self-hosted to avoid usage limits

* Bugfixes for build-artifacts workflow

* If not in CI and env var unset, don't pin common

* Style: make 'click' install requirement lowercase

.github/workflows/build-artifacts.yml

@@ -1,45 +1,101 @@
-name: Build artifacts
+name: Build Python artifacts
 
-on: [workflow_call, workflow_dispatch]
+on:
+  workflow_call:
+    inputs:
+      artifact-name:
+        required: true
+        type: string
+      source-path:
+        required: true
+        type: string
+      setup-file:
+        required: true
+        type: string
+      watched-paths:
+        required: true
+        type: string
 
-jobs:
-  bugswarm-common:
-    name: Package bugswarm.common
-    runs-on: ubuntu-latest
+  workflow_dispatch:
+    inputs:
+      artifact-name:
+        required: true
+        type: string
+      source-path:
+        required: true
+        type: string
+      setup-file:
+        required: true
+        type: string
 
+jobs:
+  check-changed-files:
+    name: Check for changed files
+    runs-on: ${{ (github.repository == 'BugSwarm/bugswarm-dev' && 'self-hosted') || 'ubuntu-latest' }}
+    container: bugswarm/images:ubuntu-20.04
+    outputs:
+      any-changed: ${{ github.event_name == 'workflow_dispatch' || steps.changed-files.outputs.any_changed == 'true'}}
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
       - name: Get changed files list
         uses: tj-actions/changed-files@v34
         id: changed-files
         with:
-          files: |
-            bugswarm/common/**
-            setup.common.py
+          files: ${{ inputs.watched-paths }}
 
-      - name: Prepare repo for packaging
-        if: steps.changed-files.outputs.any_changed == 'true'
+  build-artifacts:
+    name: Package ${{ inputs.artifact-name }}
+    runs-on: ${{ (github.repository == 'BugSwarm/bugswarm-dev' && 'self-hosted') || 'ubuntu-latest' }}
+    container: bugswarm/images:ubuntu-20.04
+    needs: check-changed-files
+    if: needs.check-changed-files.outputs.any-changed == 'true'
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # - uses: actions/setup-python@v4
+      #   with:
+      #     python-version: '3.8'
+
+      # Make sure that the python build package works (only needed if we're in the container)
+      - run: sudo apt update; sudo apt install -y python3.8-venv
+
+      # If we're building bugswarm-common, modify credentials.py
+      - name: Modify credentials (bugswarm-common only)
+        if: inputs.artifact-name == 'bugswarm-common-build'
+        working-directory: bugswarm/common
         run: |
-          pushd bugswarm/common
           mv credentials.sample.py credentials.py
           sed -i "s/COMMON_HOSTNAME = ''/COMMON_HOSTNAME = 'www.api.bugswarm.org'/g" credentials.py
           sed -i "s/\(.*\) = \(''\|\[\]\)/\1 = '#'/g" credentials.py
-          popd
-          mv bugswarm/common/README.md README.md
-          mv setup.common.py setup.py
+
+      # If we're building bugswarm-client, check for a new version of bugswarm-common.
+      - name: Check for bugswarm-common artifact (bugswarm-client only)
+        id: bugswarm-common-artifact
+        if: inputs.artifact-name == 'bugswarm-client-build'
+        uses: actions/download-artifact@v3
+        with:
+          name: bugswarm-common-build
+          path: tmp
+        continue-on-error: true
+
+      - name: Prepare repo for packaging
+        env:
+          SOURCE_PATH: ${{ inputs.source-path }}
+          SETUP_FILE: ${{ inputs.setup-file }}
+        run: |
+          mv "${SOURCE_PATH}/README.md" README.md
+          mv "$SETUP_FILE" setup.py
       - name: Build wheel and source
-        if: steps.changed-files.outputs.any_changed == 'true'
+        env:
+          BSC_UPDATED: ${{ steps.bugswarm-common-artifact.outcome == 'success' || '' }}
         run: |
-          python3 -m pip install build
-          python3 -m build --sdist --wheel --outdir dist
+          python3.8 -m pip install build requests
+          python3.8 -m build --sdist --wheel --outdir dist
       - name: Upload dists
-        if: steps.changed-files.outputs.any_changed == 'true'
         uses: actions/upload-artifact@v3
         with:
-          name: bugswarm-common-build
+          name: ${{ inputs.artifact-name }}
           path: ./dist

.github/workflows/main.yml

@@ -63,11 +63,33 @@ jobs:
         run: |
           python3.8 -m unittest discover
 
-  bugswarm_dev_build:
-    name: Build artifacts
+  bugswarm_dev_build_common:
+    name: Build bugswarm/common
     if: github.event_name == 'pull_request'
     needs: bugswarm_dev
     uses: ./.github/workflows/build-artifacts.yml
+    with:
+      artifact-name: bugswarm-common-build
+      source-path: bugswarm/common
+      setup-file: setup.common.py
+      watched-paths: |
+        bugswarm/common/**
+        setup.common.py
+  
+  bugswarm_dev_build_client:
+    name: Build bugswarm/client
+    if: github.event_name == 'pull_request'
+    needs: bugswarm_dev_build_common
+    uses: ./.github/workflows/build-artifacts.yml
+    with:
+      artifact-name: bugswarm-client-build
+      source-path: bugswarm/client
+      setup-file: setup.client.py
+      watched-paths: |
+        bugswarm/common/**
+        bugswarm/client/**
+        setup.common.py
+        setup.client.py
 
   bugswarm:
     name: Test in public repo
@@ -123,8 +145,30 @@ jobs:
         run: |
           python3 -m unittest discover
 
-  bugswarm_build:
-    name: Build artifacts
+  bugswarm_build_common:
+    name: Build bugswarm/common
     if: github.event_name == 'push' && github.ref_name == 'master'
     needs: bugswarm
     uses: ./.github/workflows/build-artifacts.yml
+    with:
+      artifact-name: bugswarm-common-build
+      source-path: bugswarm/common
+      setup-file: setup.common.py
+      watched-paths: |
+        bugswarm/common/**
+        setup.common.py
+
+  bugswarm_build_client:
+    name: Build bugswarm/client
+    if: github.event_name == 'push' && github.ref_name == 'master'
+    needs: bugswarm_build_common
+    uses: ./.github/workflows/build-artifacts.yml
+    with:
+      artifact-name: bugswarm-client-build
+      source-path: bugswarm/client
+      setup-file: setup.client.py
+      watched-paths: |
+        bugswarm/common/**
+        bugswarm/client/**
+        setup.common.py
+        setup.client.py

.github/workflows/publish.yml

@@ -8,8 +8,12 @@ on:
 
 jobs:
   bugswarm-common:
-    name: Publish bugswarm.common to PyPI
+    name: Publish ${{ matrix.artifact-name }} to PyPI
     runs-on: ubuntu-latest
+    matrix:
+      artifact-name:
+        - bugswarm-common-build
+        - bugswarm-client-build
 
     steps:
       # Because actions/download-artifact can only download artifacts from the same workflow run,
@@ -20,12 +24,15 @@ jobs:
         # We don't want to fail the entire job if there aren't any new artifacts.
         # Make sure that all later steps check that steps.get-download-url.outcome == 'success'.
         continue-on-error: true
+        env:
+          ARTIFACT_NAME: ${{ matrix.artifact-name }}
         with:
           result-encoding: string
           script: |
+            const { ARTIFACT_NAME } = process.env;
             const mostRecentArtifact = (await github.rest.actions.listArtifactsForRepo({
               ...context.repo,
-              name: "bugswarm-common-build",
+              name: ARTIFACT_NAME,
             })).data.artifacts[0];
 
             const creationTime = new Date(mostRecentArtifact.created_at);

bugswarm/client/README.md

@@ -0,0 +1,69 @@
+# The BugSwarm Client
+
+![The BugSwarm Mascot](https://cloud.githubusercontent.com/assets/8139148/24324903/1101b9a2-114c-11e7-9340-316022ef57d5.png)
+
+The official command line client for the [BugSwarm](https://bugswarm.org) artifact dataset
+
+## Installation
+> Requires Python 3.
+```
+$ pip3 install bugswarm-client
+```
+
+## Usage
+Download a Docker image and enter the Docker container associated with an artifact.
+```shell
+$ bugswarm run --image-tag <image_tag>
+```
+> Depending on how Docker is configured on your machine, you may need to enter an administrator password.
+
+Download a Docker image and enter the Docker container with a shared folder between the container and the host machine.
+
+```shell
+$ bugswarm run --image-tag <image_tag> --use-sandbox
+```
+
+Show metadata for an artifact.
+
+```shell
+$ bugswarm show --image-tag <image_tag> --token <token>
+```
+
+Show usage text for the entire tool or for a specific sub-command.
+
+```shell
+$ bugswarm --help
+$ bugswarm <sub-command> --help
+```
+
+Show the version.
+
+```shell
+$ bugswarm --version
+```
+
+Please note that artifacts are first attempted to be pulled from `bugswarm/cached-images`, and if not found then they are attempted to be pulled from `bugswarm/images`.
+
+## Example
+
+```shell
+$ bugswarm run --image-tag nutzam-nutz-140438299
+$ bugswarm show --image-tag nutzam-nutz-140438299 --token <token>
+```
+
+> [Requires a token](http://www.bugswarm.org/contact/)
+
+## Development
+Execute the following commands to install the tool in ["editable" mode](https://pip.pypa.io/en/stable/cli/pip_install/#editable-installs).
+1. Clone this repository.
+    ```
+    $ git clone https://github.com/BugSwarm/client.git
+    ```
+1. `cd` into the root directory of this repository.
+    ```
+    $ cd client
+    ```
+1. Install the tool.
+    ```
+    $ pip3 install --upgrade --force-reinstall -e .
+    ```

bugswarm/client/bugswarm.py

@@ -0,0 +1,62 @@
+import json
+import logging
+import os
+
+import click
+
+from bugswarm.common import log
+from bugswarm.common.rest_api.database_api import DatabaseAPI
+
+from . import docker
+from .command import MyCommand
+
+
+@click.group()
+@click.version_option(message='The BugSwarm Client, version %(version)s')
+def cli():
+    """A command line interface for the BugSwarm dataset."""
+    # Configure logging.
+    log.config_logging(getattr(logging, 'INFO', None))
+
+
+@cli.command(cls=MyCommand)
+@click.option('--image-tag', required=True,
+              type=str,
+              help='The artifact image tag.')
+@click.option('--use-sandbox/--no-use-sandbox', default=False,
+              help='Whether to set up a directory that is shared by the host and container.')
+@click.option('--pipe-stdin/--no-pipe-stdin', default=False,
+              help='If enabled, the contents of stdin are executed inside the container. '
+                   'This option supports heredocs in shells that support them. '
+                   'Disabled by default.')
+@click.option('--rm/--no-rm', default=True,
+              help='If enabled, artifact containers will be cleaned up automatically after use. '
+                   'Disable this behavior if you want to inspect the container filesystem after use. '
+                   'Enabled by default.')
+def run(image_tag, use_sandbox, pipe_stdin, rm):
+    """Start an artifact container."""
+    # If the script does not already have sudo privileges, then explain to the user why the password prompt will appear.
+    if os.getuid() != 0:
+        log.info('Docker requires sudo privileges.')
+    docker.docker_run(image_tag, use_sandbox, pipe_stdin, rm)
+
+
+@cli.command(cls=MyCommand)
+@click.option('--image-tag', required=True,
+              type=str,
+              help='The artifact image tag.')
+@click.option('--token', required=True,
+              type=str,
+              help='An authentication token for the BugSwarm database. '
+                   'Please visit www.bugswarm.org/get-full-access for more information.')
+def show(image_tag, token):
+    """Display artifact metadata."""
+    token = token or ''
+    bugswarmapi = DatabaseAPI(token=token)
+    response = bugswarmapi.find_artifact(image_tag, error_if_not_found=False)
+    if not response.ok:
+        log.info('No artifact metadata found for image tag {}.'.format(image_tag))
+    else:
+        artifact = response.json()
+        # Print without the INFO prefix so the output is easier to parse.
+        print(json.dumps(artifact, sort_keys=True, indent=4))

bugswarm/client/command.py

@@ -0,0 +1,17 @@
+from bugswarm.common import outdated
+from click import Command
+
+
+class MyCommand(Command):
+    """
+    A subclass of Click's Command class that checks if the client is outdated after invoking the command.
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def invoke(self, ctx):
+        try:
+            super().invoke(ctx)
+        finally:
+            # Ask users to consider updating if a newer version of the client is available.
+            outdated.check_package_outdated('bugswarm-client')