GraphDB storage POC

.gitignore

@@ -1,2 +1,4 @@
 .idea
 venv/
+pgdata/
+__pycache__/

README.MD

@@ -1,110 +1,34 @@
-# Vantage6 installation
+# Vantage 6 local setup
 
-Install:
-```shell
-virtualenv venv
-source venv/bin/activate
-pip install vantage6
-```
-
-Configure server:
-```shell
-# --user parameter makes sure config is stored in user directory, not system wide
-vserver new --user # Wizard prompts you for basic info
+## Running the local setup
+Simply run:
+```shell script
+docker-compose up --force-recreate
 
-vserver start --user
+# Populate 'escience' node rdf store with sample data
+./nodes/escience/populate-rdf-store.sh
 ```
 
-At the first run, a new user will be created:
-```
-username: root
-password: root
+# Executing a task
+When your docker setup is running you can execute a task using `run_task.py`. Replace the value in `IMAGE` with your
+desired docker image
 
+# Docker registry
+For development purposes you might want to run your own local docker registry:
 ```
-In order to get access to the REST api you will need to request a user token:
-
-```shell
-# Get your user token
-curl --location --request POST 'localhost:5000/api/token/user' \
---header 'Content-Type: application/json' \
---data-raw '{
-  "username": "root",
-  "password": "root"
-}'
-
+# Run docker registry separately. Node tasks are run in a network separate from the node itself. By making the registry
+# available from the host network it will be accessable by all docker containers
+docker run -d  --name registry --network host registry:2
 ```
 
-Now you can configure your organization, collaboration and node.
-This can also be done by importing a configuration file, but using the REST api ensures
-the input is validated, which is nice when trying out vantage6 for the first time.
-
-Make sure to replace `YOUR_API_KEY` with the key received in the previous step.
-
-```shell
-# Organization
-
-curl --location --request POST 'localhost:5000/api/organization' \
---header 'Authorization: Bearer YOUR_API_KEY' \
---header 'Content-Type: application/json' \
---data-raw '{
-  "address1": "Some Street 11, Amsterdam",
-  "country": "Netherlands",
-  "name": "escience center",
-  "zipcode": "1234ab"
-}'
-
-# Collaboration
-curl --location --request POST 'localhost:5000/api/collaboration' \
---header 'Authorization: Bearer YOUR_API_KEY' \
---header 'Content-Type: application/json' \
---data-raw '{
-    "name": "collab1",
-    "organization_ids": [
-        1
-    ]
-}'
-
-# Add organization to collaboration
-
-curl --location --request POST 'localhost:5000/api/collaboration/1/organization' \
---header 'Authorization: Bearer YOUR_API_KEY' \
---header 'Content-Type: application/json' \
---data-raw '{"id": 1}'
-
-
-# Create user tied to organization
-
-curl --location --request POST 'localhost:5000/api/user' \
---header 'Authorization: Bearer YOUR_API_KEY' \
---header 'Content-Type: application/json' \
---data-raw '{
-
-  "firstname": "djura",
-  "lastname": "smits",
-  "password": "admin",
-  "organization_id": "1",
-  "username": "admin",
-  "roles": ["admin"]
-
-}'
-
-# Now retrieve token for this new user
-#
-#
-#
-
-# Node
-# Create node
-curl --location --request POST 'localhost:5000/api/node' \
---header 'Authorization: Bearer YOUR_API_KEY' \
---header 'Content-Type: application/json' \
---data-raw '{
-	"collaboration_id": 1
-}'
+The `run_task.py` script runs a custom algorithm [available on github](https://github.com/CARRIER-project/vantage6-algorithms)
+Make sure this task is published to the local registry by running `push_to_local_registry.sh` from that repo
 
 ```
-Check the api key that you have received and use that for your configuration of the vnode.
+python run_task.py
+```
 
-```bash
-vnode new # Follow wizard and specify api key
+To run the sample_sparqle_query method:
+```
+python run_task.py --rpc --method 'sample_sparqle_query'
 ```

add_new_node.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+import click
+
+from carrier import VantageClient
+
+USERNAME = 'root'
+PASSWORD = 'root'
+ADMIN_PASSWORD = 'admin'
+COLLABORATION_ID = 1
+
+ORGANIZATION_BASE = ORGANIZATION = {'address1': 'my address 1, Amsterdam',
+                                    'country': 'the Netherlands',
+                                    'zipcode': '1234ab'}
+
+
+@click.command()
+@click.argument('name')
+@click.option('--org-id')
+@click.option('--username')
+@click.option('--password', default=ADMIN_PASSWORD)
+def create_node(name, org_id, username, password):
+
+    if not (org_id and username):
+
+        client = VantageClient(USERNAME, PASSWORD)
+
+    if not org_id:
+        print('Creating new organization')
+        org_id = create_organization(client, name)
+
+    if not username:
+        username = f'admin_{org_id}'
+        print(f'Creating new user {username}')
+        # Create user for organization
+        create_user(client, org_id, username)
+
+    # Create node
+    client = VantageClient(username, password)
+    result = client.post('node', {'collaboration_id': COLLABORATION_ID, 'organization_id': org_id})
+    api_key = result['api_key']
+
+    print(f'Created new node. Api key: {api_key}')
+
+
+def create_organization(client, name):
+    # Create organization for node
+    organization = dict(ORGANIZATION_BASE)
+    organization['name'] = name
+    result = client.post('organization', organization)
+    org_id = result['id']
+    return org_id
+
+
+def create_user(client, organization_id, username):
+    user = {'firstname': ' ', 'lastname': ' ', 'username': username, 'organization_id': organization_id,
+            'password': ADMIN_PASSWORD, 'roles': ['admin']}
+    result = client.post('user', user)
+    print(f'Created new user:\n{result}')
+
+
+if __name__ == '__main__':
+    create_node()

carrier/__init__.py

@@ -0,0 +1 @@
+from .vantage_client import *

carrier/vantage_client.py

@@ -0,0 +1,89 @@
+import base64
+import json
+import pickle
+
+import requests
+from vantage6.client import Client
+
+_HOST = 'http://localhost'
+_PORT = 5001
+
+HOST = 'localhost'
+PORT = 5001
+PREFIX = 'api'
+DEFAULT_SERVER_ROOT = f'{HOST}:{PORT}/{PREFIX}/'
+OK_RESPONSES = [200, 201]
+DEFAULT_CONTENT_TYPE = 'application/json'
+POST = 'POST'
+DEFAULT_HEADERS = {'Content-Type': DEFAULT_CONTENT_TYPE}
+
+
+def get_official_client(username, password):
+    """
+    Get official vantage6 client.
+
+    :param username:
+    :param password:
+    :return:
+    """
+    client = Client(_HOST, _PORT)
+    client.authenticate(username, password)
+    client.setup_encryption(None)
+
+    return client
+
+
+class VantageClient():
+    """
+    Custom made vantage client to work around some problems the official has at the moment (such as authenticating root
+    users).
+    """
+
+    def __init__(self, username, password):
+        # Retrieve a authentication token
+        self.token = self.get_token(username, password)
+        self.headers = {
+            'Authorization': f'Bearer {self.token}',
+            'Content-Type': DEFAULT_CONTENT_TYPE
+        }
+
+    @staticmethod
+    def get_url(endpoint) -> str:
+        return 'http://' + DEFAULT_SERVER_ROOT + endpoint
+
+    def get_token(self, username, password):
+        result = self.request('token/user', {'username': username, 'password': password}, headers=DEFAULT_HEADERS,
+                              method=POST)
+        return result['access_token']
+
+    def get(self, endpoint, payload=None, headers=None) -> dict:
+        return self.request(endpoint, payload, headers, 'GET')
+
+    def post(self, endpoint, payload, headers=None):
+        print(f'Posting: {payload}')
+        return self.request(endpoint, payload, headers, 'POST')
+
+    def post_task(self, name, image, collaboration_id, organizations):
+        for o in organizations:
+            input_base64 = base64.b64encode(pickle.dumps(o['input']))
+            o['input'] = str(input_base64, 'utf8')
+            print(f'Base64 converted input: {o}')
+
+        payload = {'collaboration_id': collaboration_id, 'image': image, 'name': name, 'organizations': organizations}
+        return self.post('task', payload)
+
+    def request(self, endpoint, payload, headers=None, method='GET') -> dict:
+        if headers is None:
+            headers = self.headers
+
+        url = VantageClient.get_url(endpoint)
+
+        print(f'Request {method} {url}')
+
+        headers['Content-Type'] = 'application/json'
+        response = requests.request(method, url, headers=headers, data=json.dumps(payload))
+
+        if response.status_code in OK_RESPONSES:
+            return response.json()
+        else:
+            raise Exception(f'Request returned status {response.status_code}\nMessage: {response.content}')