From c8e123d0aa1458cf88a1a5eea3a7b3a375c21ace Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 23 Aug 2024 16:33:39 -0500 Subject: [PATCH] Initial commit --- .github/workflows/build-and-release.yml | 112 ++++++++++++++++++ .gitignore | 7 ++ Dockerfile | 12 ++ README.md | 145 ++++++++++++++++++++++++ config.yaml | 55 +++++++++ demo/input/user123.json | 3 + demo/input/user456.json | 3 + my_proof/__init__.py | 0 my_proof/__main__.py | 3 + my_proof/proof.py | 54 +++++++++ python.manifest.template | 26 +++++ 11 files changed, 420 insertions(+) create mode 100644 .github/workflows/build-and-release.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 config.yaml create mode 100644 demo/input/user123.json create mode 100644 demo/input/user456.json create mode 100644 my_proof/__init__.py create mode 100644 my_proof/__main__.py create mode 100644 my_proof/proof.py create mode 100644 python.manifest.template diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 0000000..40a0913 --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,112 @@ +name: Build and Release + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: write + +jobs: + build-and-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build Docker image + uses: docker/build-push-action@v4 + with: + context: . + load: true + tags: | + my-proof:${{ github.run_number }} + my-proof:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Clone and set up GSC + run: | + git clone https://github.com/gramineproject/gsc.git + cd gsc + python3 -m pip install --no-cache-dir 'docker>=7.1.0' 'jinja2>=3.1.4' 'tomli>=2.0.1' 'tomli-w>=1.0.0' 'pyyaml>=6.0.2' + + - name: Create signing key + run: | + echo "${{ secrets.SIGNING_KEY }}" > signing_key.pem + chmod 600 signing_key.pem + + - name: Build GSC image + run: | + cd gsc + ./gsc build my-proof ../python.manifest.template -c ../config.yaml + + - name: Sign GSC image + run: | + cd gsc + ./gsc sign-image my-proof ../signing_key.pem -c ../config.yaml + + - name: Export GSC image to file + run: | + docker save gsc-my-proof:latest | gzip > gsc-my-proof-${{ github.run_number }}.tar.gz + + - name: Generate verification data + run: | + cd gsc + ./gsc info-image gsc-my-proof > ../sigstruct.txt + + - name: Upload image + uses: actions/upload-artifact@v3 + with: + name: gsc-my-proof-image + path: gsc-my-proof-${{ github.run_number }}.tar.gz + + - name: Upload verification data + uses: actions/upload-artifact@v3 + with: + name: gsc-my-proof-sigstruct + path: sigstruct.txt + + - name: Generate release body + run: | + echo "MRSIGNER: $(grep -oP 'mr_signer = "\K[^"]*' sigstruct.txt)" >> release_body.txt + echo "MRENCLAVE: $(grep -oP 'mr_enclave = "\K[^"]*' sigstruct.txt)" >> release_body.txt + echo "Image SHA256: $(sha256sum gsc-my-proof-${{ github.run_number }}.tar.gz | cut -d' ' -f1)" >> release_body.txt + + - name: Create Release and Upload Assets + uses: softprops/action-gh-release@v1 + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ github.run_number }} + name: Release v${{ github.run_number }} + body_path: release_body.txt + draft: false + prerelease: false + files: | + ./gsc-my-proof-${{ github.run_number }}.tar.gz + ./sigstruct.txt + + - name: Cleanup signing key + if: always() + run: | + rm -f signing_key.pem + + - name: Log build result + if: always() + run: | + if [ ${{ job.status }} == "success" ]; then + echo "Build and release completed successfully" + else + echo "Build and release failed" + fi \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cbe5627 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.pem +*.tar.gz +__pycache__/ +*.pyc + +demo/* +!demo/input/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0ae96eb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +# Install any Python dependencies your application needs, e.g.: +# RUN pip install --no-cache-dir cryptography + +RUN mkdir /sealed && chmod 777 /sealed + +WORKDIR /app + +COPY . /app + +CMD ["python", "-m", "my_proof"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..0818de8 --- /dev/null +++ b/README.md @@ -0,0 +1,145 @@ +# Vana Satya Proof of Contribution Python Template + +This repository serves as a template for creating proof tasks using Python and Gramine for secure computation. + +## Overview + +This template provides a basic structure for building proof tasks that: + +1. Read input files from the `/input` directory +2. Process the data securely +3. Write proof results to the `/output` directory +4. Use the `/sealed` directory for secure storage (when available) + +The project is designed to work with Gramine, a lightweight library OS that enables running unmodified applications in secure enclaves, such as Intel SGX (Software Guard Extensions). This allows the code to run in a trusted execution environment, ensuring confidentiality and integrity of the computation. + +## Project Structure + +- `my_proof/`: Contains the main proof logic + - `proof.py`: Implements the proof generation logic + - `__main__.py`: Entry point for the proof execution +- `demo/`: Contains sample input and output for testing +- `.github/workflows/`: CI/CD pipeline for building and releasing +- `Dockerfile`: Defines the container image for the proof task +- `python.manifest.template`: Gramine manifest template for secure execution +- `config.yaml`: Configuration file for Gramine Shielded Containers (GSC) + +## Getting Started + +To use this template: + +1. Fork this repository +2. Modify the `my_proof/proof.py` file to implement your specific proof logic +3. Update the `python.manifest.template` if you need to add any additional files or change the configuration +4. Commit your changes and push to your repository + +## Customizing the Proof Logic + +The main proof logic is implemented in `my_proof/proof.py`. To customize it: + +1. Modify the `expected_words` list to include the keywords you're looking for +2. Adjust the `process_on_disk` and `process_in_memory` functions to implement your specific proof generation logic +3. Update the `generate_proof` function if you need to change how input files are processed or results are written + +## Local Development + +To run the proof locally, without Gramine, you can use Docker: + +``` +docker build -t my-proof . +docker run \ +--rm \ +--volume $(pwd)/demo/sealed:/sealed \ +--volume $(pwd)/demo/input:/input \ +--volume $(pwd)/demo/output:/output \ +my-proof +``` + +## Generating the Gramine Signing Key + +Before building and signing your graminized Docker image, you need to generate a signing key. Gramine provides a tool called `gramine-sgx-gen-private-key` for this purpose. Here's how to use it: + +1. If you have Gramine installed on your system, you can use the following command: + + ``` + gramine-sgx-gen-private-key enclave-key.pem + ``` + + This will generate a new RSA 3072 key suitable for signing SGX enclaves and save it as `enclave-key.pem` in the current directory. + +2. If you don't have Gramine installed, you can use OpenSSL to generate a compatible key: + + ``` + openssl genrsa -3 -out enclave-key.pem 3072 + ``` + + This generates an RSA 3072-bit key with the public exponent set to 3, which is required for SGX enclaves. + +Make sure to keep this key secure, as it will be used to sign your enclaves. You'll use this key in the `gsc sign-image` step of the GSC workflow. + +## Building and Releasing + +This template includes a GitHub Actions workflow that automatically: + +1. Builds a Docker image with your code +2. Creates a Gramine-shielded container (GSC) image +3. Publishes the GSC image as a GitHub release + +To use this workflow, you need to: + +1. Set up a signing key, as described above, and add it as a GitHub secret named `SIGNING_KEY` +2. Push your changes to the `main` branch or create a pull request + +## Running with SGX + +Intel SGX (Software Guard Extensions) is a set of security-related instruction codes built into modern Intel CPUs. It allows parts of a program to be executed in a secure enclave, isolated from the rest of the system. + +To load a released image with docker, copy the URL from the release and run: + +``` +curl -L https://github.com/vana-com/vana-satya-proof-template-temporary/releases/download/v9/gsc-my-proof-latest.tar.gz | docker load +``` + +To run the image: + +``` +docker run \ +--rm \ +--volume /gsc-my-proof/input:/input \ +--volume /gsc-my-proof/output:/output \ +gsc-my-proof +``` + +If your application relies on Gramine-enabled SGX features like sealing, remote attestation, etc., you may need to mount additional volumes and devices: + +``` +docker run \ +--rm \ +--volume /gsc-my-proof/input:/input \ +--volume /gsc-my-proof/output:/output \ +--device /dev/sgx_enclave:/dev/sgx_enclave \ +--volume /var/run/aesmd:/var/run/aesmd \ +--volume /mnt/gsc-my-proof/sealed:/sealed \ +gsc-my-proof +``` + +## Security Features + +This template leverages several security features: + +1. **Secure Enclaves**: The proof runs inside an SGX enclave, isolating it from the rest of the system. +2. **Encrypted Storage**: The `/sealed` directory is automatically encrypted/decrypted by Gramine, providing secure storage for sensitive data. +3. **Input/Output Isolation**: Input and output directories are mounted separately, ensuring clear data flow boundaries. +4. **Minimal Attack Surface**: The Gramine manifest limits the files and resources accessible to the enclave, reducing potential vulnerabilities. + +## Customization + +Feel free to modify any part of this template to fit your specific needs. The goal is to provide a starting point that can be easily adapted to various proof tasks. + +## Contributing + +If you have suggestions for improving this template, please open an issue or submit a pull request. + +## License + +[MIT License](LICENSE) \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..04270db --- /dev/null +++ b/config.yaml @@ -0,0 +1,55 @@ +# NOTE: This file was copied from https://github.com/gramineproject/gsc/blob/fcf96546f4a23a4e6bcc6a14d80cf1521c018fc9/config.yaml.template + +# +# Specify the OS distro that is used to build Gramine, i.e., the distro from where the Gramine build +# gets all tools and dependencies from. This distro should match the distro underlying the +# application's Docker image; otherwise the results may be unpredictable (if you specify `"auto"`, +# which is recommended, you don't need to worry about the mismatch). +# +# Currently supported distros are: +# - ubuntu:20.04, ubuntu:21.04, ubuntu:22.04, ubuntu:23.04 +# - debian:10, debian:11, debian:12 +# - centos:8 +# - quay.io/centos/centos:stream9 +# - redhat/ubi8:8.8, redhat/ubi9:9.4 +# - redhat/ubi8-minimal:8.8, redhat/ubi9-minimal:9.4 + +# If Distro is set to "auto", GSC detects the distro automatically by examining the supplied +# Docker image. Alternatively, Distro can be set to one of the supported distros mentioned above. +Distro: "auto" + +# If the image has a specific registry, define it here. +# Empty by default; example value: "registry.access.redhat.com/ubi8". +Registry: "" + +# If you're using your own fork and branch of Gramine, specify the GitHub link and the branch name +# below; typically, you want to keep the default values though. +# +# It is also possible to specify the prebuilt Gramine Docker image (that was built previously via +# the `gsc build-gramine` command). For this, remove Repository and Branch and instead write: +# Image: "" +# +# GSC releases are guaranteed to work with corresponding Gramine releases (and GSC `master` +# branch is guaranteed to work with current Gramine `master` branch). +Gramine: + Repository: "https://github.com/gramineproject/gramine.git" + Branch: "master" + +# Specify the Intel SGX driver installed on your machine (more specifically, on the machine where +# the graminized Docker container will run); there are several variants of the SGX driver: +# +# - upstream (in-kernel) driver: use empty values like below +# Repository: "" +# Branch: "" +# +# - DCAP out-of-tree driver: same as above, use empty values +# Repository: "" +# Branch: "" +# +# - legacy out-of-tree driver: use something like the below values, but adjust the branch name +# Repository: "https://github.com/01org/linux-sgx-driver.git" +# Branch: "sgx_driver_1.9" +# +SGXDriver: + Repository: "" + Branch: "" \ No newline at end of file diff --git a/demo/input/user123.json b/demo/input/user123.json new file mode 100644 index 0000000..6a0c96e --- /dev/null +++ b/demo/input/user123.json @@ -0,0 +1,3 @@ +{ + "data": "hello world" +} diff --git a/demo/input/user456.json b/demo/input/user456.json new file mode 100644 index 0000000..d351dd8 --- /dev/null +++ b/demo/input/user456.json @@ -0,0 +1,3 @@ +{ + "data": "goodnight world" +} diff --git a/my_proof/__init__.py b/my_proof/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/my_proof/__main__.py b/my_proof/__main__.py new file mode 100644 index 0000000..2c6e4d5 --- /dev/null +++ b/my_proof/__main__.py @@ -0,0 +1,3 @@ +from .proof import generate_proof + +generate_proof() \ No newline at end of file diff --git a/my_proof/proof.py b/my_proof/proof.py new file mode 100644 index 0000000..8be1486 --- /dev/null +++ b/my_proof/proof.py @@ -0,0 +1,54 @@ +import os +import json + +INPUT_DIR = '/input' +OUTPUT_DIR = '/output' +SEALED_DIR = '/sealed' + +expected_words = ["hello", "world"] + +# Gramine will auto encrypt/decrypt anything in SEALED_DIR +def process_on_disk(input_file): + with open(input_file, 'r') as f: + data = json.load(f).get('data', '') + + sealed_file = os.path.join(SEALED_DIR, f'{os.path.basename(input_file)}_sealed.txt') + + with open(sealed_file, 'w') as sf: + for word in expected_words: + if word in data: + sf.write(f"{word}\n") + + with open(sealed_file, 'r') as sf: + found_count = sum(1 for _ in sf) + + return {"valid_contribution": found_count == len(expected_words)} + +def process_in_memory(input_file): + with open(input_file, 'r') as f: + data = json.load(f).get('data', '') + + found_keywords = set() + found_keywords.update(word for word in expected_words if word in data) + + return {"valid_contribution": len(found_keywords) == len(expected_words)} + +def generate_proof(): + use_sealing = os.path.isdir(SEALED_DIR) + + for filename in os.listdir(INPUT_DIR): + input_path = os.path.join(INPUT_DIR, filename) + output_path = os.path.join(OUTPUT_DIR, f"{filename}_result.json") + + if use_sealing: + result = process_on_disk(input_path) + else: + result = process_in_memory(input_path) + + with open(output_path, 'w') as f: + json.dump(result, f, indent=2) + + +if __name__ == "__main__": + generate_proof() + print("Proof generation complete. Check the output directory for results.") \ No newline at end of file diff --git a/python.manifest.template b/python.manifest.template new file mode 100644 index 0000000..3f3c139 --- /dev/null +++ b/python.manifest.template @@ -0,0 +1,26 @@ +# Adjust this as needed. +sgx.enclave_size = "256M" + +# Increase this as needed, e.g., if you run a web server. +sgx.max_threads = 2 + +fs.mounts = [ + { type = "encrypted", path = "/sealed", uri = "file:/sealed", key_name = "_sgx_mrenclave" }, + { path = "/input", uri = "file:/input" }, + { path = "/output", uri = "file:/output" }, +] + +# Gramine gives a warning that allowed_files is not safe in production, but it +# should generally be fine for our use case which inherently assumes that input +# files are untrusted until proven otherwise. +# +# Establishing a secure channel would be strong in that it would be harder for +# malicious third parties to tamper with the files. +sgx.allowed_files = [ + "file:/input/", + "file:/output/", +] + +# You can add other Gramine-manifest-compatible options as needed, see the +# Gramine documentation for more details: https://gramine.readthedocs.io. Note +# that gsc defines a number of manifest settings by default. \ No newline at end of file