Skip to content

Commit

Permalink
Merge pull request #184 from catenax-ng/feature/97-ExternalMatchmaking
Browse files Browse the repository at this point in the history
changes added for feature 97 MarchmakingAgentApi
  • Loading branch information
almadigabor authored Apr 16, 2024
2 parents 3072a10 + a38fbe0 commit 6917953
Show file tree
Hide file tree
Showing 38 changed files with 397 additions and 94 deletions.
32 changes: 0 additions & 32 deletions .github/actions/setup-java/action.yml

This file was deleted.

13 changes: 9 additions & 4 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,13 @@ jobs:
with:
submodules: recursive

# Setup build environment
- uses: ./.github/actions/setup-java
# Set-Up
- name: Setup JDK 17
uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
with:
java-version: '17'
distribution: 'temurin'
cache: 'maven'

# Enable deployment access (on demand or main branch and version tags only)
- name: Login to GitHub Container Registry
Expand Down Expand Up @@ -137,7 +142,7 @@ jobs:
type=semver,pattern={{version}}
type=semver,pattern={{major}}
type=semver,pattern={{major}}.{{minor}}
type=raw,value=1.12.17-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }}
type=raw,value=1.12.18-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }}
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
- name: Agent Plane Hashicorp Container Build and push
Expand Down Expand Up @@ -175,7 +180,7 @@ jobs:
type=semver,pattern={{version}}
type=semver,pattern={{major}}
type=semver,pattern={{major}}.{{minor}}
type=raw,value=1.12.17-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }}
type=raw,value=1.12.18-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }}
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
- name: Agent Plane Azure Vault Container Build and push
Expand Down
98 changes: 98 additions & 0 deletions .github/workflows/dependencies.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
###############################################################
# Copyright (c) 2021, 2024 Contributors to the Eclipse Foundation
#
# See the NOTICE file(s) distributed with this work for additional
# information regarding copyright ownership.
#
# This program and the accompanying materials are made available under the
# terms of the Apache License, Version 2.0 which is available at
# https://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# SPDX-License-Identifier: Apache-2.0
###############################################################

name: Check Dependencies

on:
push:
branches:
- main
- 'release/*'
pull_request:
branches:
- main
- 'release/*'
types:
- opened
- synchronize
- reopened
workflow_dispatch:

jobs:
check-dependencies:

runs-on: ubuntu-latest
strategy:
matrix:
dotnet-version: ['7.0']

steps:

# Get the Code
- name: Checkout repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
submodules: recursive

# Set-Up
- name: Setup JDK 17
uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
with:
java-version: '17'
distribution: 'temurin'
cache: 'maven'

# Run Maven Deploy (on demand or if either running on main or a version tag)
- name: Generate Dependencies file
if: ${{ ( github.event.inputs.deploy_maven == 'true' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') ) }}
run: |
./mvnw org.eclipse.dash:license-tool-plugin:license-check -Ddash.summary=DEPENDENCIES
- name: Check if dependencies were changed
id: dependencies-changed
run: |
changed=$(git diff DEPENDENCIES)
if [[ -n "$changed" ]]; then
echo "dependencies changed"
echo "changed=true" >> $GITHUB_OUTPUT
else
echo "dependencies not changed"
echo "changed=false" >> $GITHUB_OUTPUT
fi
- name: Check for restricted dependencies
run: |
restricted=$(grep ' restricted,' DEPENDENCIES || true)
if [[ -n "$restricted" ]]; then
echo "The following dependencies are restricted: $restricted"
exit 1
fi
if: steps.dependencies-changed.outputs.changed == 'true'

- name: Upload DEPENDENCIES file
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
path: DEPENDENCIES
if: steps.dependencies-changed.outputs.changed == 'true'

- name: Signal need to update DEPENDENCIES
run: |
echo "Dependencies need to be updated (updated DEPENDENCIES file has been uploaded to workflow run)"
exit 1
if: steps.dependencies-changed.outputs.changed == 'true'
19 changes: 16 additions & 3 deletions .github/workflows/veracode.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,14 @@ jobs:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 0
- uses: ./.github/actions/setup-java
# Set-Up
- name: Setup JDK 17
uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
with:
java-version: '17'
distribution: 'temurin'
cache: 'maven'

- name: Verify proper formatting
run: ./mvnw spotless:check

Expand All @@ -60,9 +67,15 @@ jobs:
variant: [ { dir: agent-plane, name: agentplane-azure-vault },
{ dir: agent-plane, name: agentplane-hashicorp } ]
steps:
# Set-Up
# Get Code
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: ./.github/actions/setup-java
# Set-Up
- name: Setup JDK 17
uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
with:
java-version: '17'
distribution: 'temurin'
cache: 'maven'
# Build
- name: Build ${{ matrix.variant.name }}
run: |-
Expand Down
10 changes: 5 additions & 5 deletions DEPENDENCIES
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,9 @@ maven/mavencentral/io.swagger.core.v3/swagger-jaxrs2-jakarta/2.2.2, Apache-2.0,
maven/mavencentral/io.swagger.core.v3/swagger-models-jakarta/2.2.2, Apache-2.0, approved, #5919
maven/mavencentral/jakarta.activation/jakarta.activation-api/2.1.0, EPL-2.0 OR BSD-3-Clause OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.jaf
maven/mavencentral/jakarta.annotation/jakarta.annotation-api/2.1.1, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.ca
maven/mavencentral/jakarta.inject/jakarta.inject-api/2.0.1, Apache-2.0, approved, clearlydefined
maven/mavencentral/jakarta.json/jakarta.json-api/2.1.1, EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0, approved, #7907
maven/mavencentral/jakarta.transaction/jakarta.transaction-api/2.0.0, EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0, approved, #7697
maven/mavencentral/jakarta.inject/jakarta.inject-api/2.0.1, Apache-2.0, approved, ee4j.cdi
maven/mavencentral/jakarta.json/jakarta.json-api/2.1.1, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.jsonp
maven/mavencentral/jakarta.transaction/jakarta.transaction-api/2.0.0, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.jta
maven/mavencentral/jakarta.validation/jakarta.validation-api/3.0.2, Apache-2.0, approved, ee4j.validation
maven/mavencentral/jakarta.ws.rs/jakarta.ws.rs-api/3.1.0, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.rest
maven/mavencentral/jakarta.xml.bind/jakarta.xml.bind-api/4.0.0, BSD-3-Clause, approved, ee4j.jaxb
Expand Down Expand Up @@ -224,8 +224,8 @@ maven/mavencentral/org.eclipse.jetty/jetty-servlet/11.0.15, EPL-2.0 OR Apache-2.
maven/mavencentral/org.eclipse.jetty/jetty-util/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty
maven/mavencentral/org.eclipse.jetty/jetty-webapp/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty
maven/mavencentral/org.eclipse.jetty/jetty-xml/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty
maven/mavencentral/org.eclipse.tractusx.agents.edc.agent-plane/agent-plane-protocol/1.12.17-SNAPSHOT, Apache-2.0, approved, automotive.tractusx
maven/mavencentral/org.eclipse.tractusx.edc/auth-jwt/1.12.17-SNAPSHOT, Apache-2.0, approved, automotive.tractusx
maven/mavencentral/org.eclipse.tractusx.agents.edc.agent-plane/agent-plane-protocol/1.12.18-SNAPSHOT, Apache-2.0, approved, automotive.tractusx
maven/mavencentral/org.eclipse.tractusx.edc/auth-jwt/1.12.18-SNAPSHOT, Apache-2.0, approved, automotive.tractusx
maven/mavencentral/org.eclipse.tractusx.edc/core-spi/0.5.3, Apache-2.0, approved, automotive.tractusx
maven/mavencentral/org.eclipse.tractusx.edc/edc-dataplane-azure-vault/0.5.3, Apache-2.0, approved, automotive.tractusx
maven/mavencentral/org.eclipse.tractusx.edc/edc-dataplane-base/0.5.3, Apache-2.0, approved, automotive.tractusx
Expand Down
31 changes: 30 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<!--
* Copyright (c) 2022,2023 Contributors to the Eclipse Foundation
* Copyright (c) 2022,2024 Contributors to the Eclipse Foundation
*
* See the NOTICE file(s) distributed with this work for additional
* information regarding copyright ownership.
Expand Down Expand Up @@ -98,6 +98,35 @@ Deployment can be done

See the [user documentation](docs/README.md) for more detailed deployment information.

#### Setup using Helm/Kind

In order to run KA-EDC applications via helm on your local machine, please make sure the following
preconditions are met.

- Have a local Kubernetes runtime ready. We've tested this setup with [KinD](https://kind.sigs.k8s.io/), but other
runtimes such
as [Minikube](https://minikube.sigs.k8s.io/docs/start/) may work as well, we just haven't tested them. All following
instructions will assume KinD.

For the most bare-bones installation of the dataspace, execute the following commands in a shell:

```shell
kind create cluster -n ka --config kind.config.yaml
# the next step is specific to KinD and will be different for other Kubernetes runtimes!
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
# wait until the ingress controller is ready
kubectl wait --namespace ingress-nginx \
--for=condition=ready pod \
--selector=app.kubernetes.io/component=controller \
--timeout=90s
# transfer images
kind load docker-image docker.io/tractusx/agentplane-hashicorp:1.12.18-SNAPSHOT --name ka
kind load docker-image docker.io/tractusx/agentplane-azure-vault:1.12.18-SNAPSHOT --name ka
# run chart testing
ct install --charts charts/agent-plane
ct install --charts charts/agent-plane-azure-vault
``````

### Notice for Docker Images

* [Notice for Agent Data Plane Running Against Hashicorp Vault](agent-plane/agentplane-hashicorp/README.md#notice-for-docker-images)
Expand Down
4 changes: 2 additions & 2 deletions agent-plane/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,10 @@ mvn package -Pwith-docker-image
Alternatively, after a successful build, you can invoke docker yourself

```console
docker build -t tractusx/agentplane-azure-vault:1.12.17-SNAPSHOT -f agentplane-azure-vault/src/main/docker/Dockerfile .
docker build -t tractusx/agentplane-azure-vault:1.12.18-SNAPSHOT -f agentplane-azure-vault/src/main/docker/Dockerfile .
```

```console
docker build -t tractusx/agentplane-hashicorp:1.12.17-SNAPSHOT -f agentplane-hashicorp/src/main/docker/Dockerfile .
docker build -t tractusx/agentplane-hashicorp:1.12.18-SNAPSHOT -f agentplane-hashicorp/src/main/docker/Dockerfile .
```

10 changes: 6 additions & 4 deletions agent-plane/agent-plane-protocol/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,12 @@ This folder hosts the [Agent Data Protocols Extension for the Eclipse Dataspace
## Architecture

This extension
- introduces a tenant-internal endpoint (Matchmaking Agent) for submitting possibly federated queries (called Skills) in the supported inference languages (currently: SparQL)
- introduces or interfaces to a tenant-internal endpoint (Matchmaking Agent) for submitting possibly federated queries (called Skills) in the supported inference languages (currently: SparQL)
- hosts a synchronisation schedule which regulary requests the catalogue from configured partner connectors and includes them into the default graph
- may negotiate further agreements for delegating sub-queries on the fly, for which purpose it also operates as an EDR callback for the control plane
- may operate as a validation proxy in case that this data plane is attached to several control planes (e.g. a consuming and a providing control plane)
- implements special Sources for dealing with http-based transfer protocols, such as SparQL-Over-Http and Skill-Over-Http
- hosts a synchronisation schedule which regulary requests the catalogue from configured partner connectors and includes them into the default graph

The SparQL implementation currently relies on Apache Jena Fuseki as the SparQL engine.

see the [Overall Source Code Layout](../../README.md#source-code-layout--runtime-collaboration)
Expand Down Expand Up @@ -63,7 +64,7 @@ Add the following dependency to your data-plane artifact pom:
<dependency>
<groupId>org.eclipse.tractusx.agents.edc</groupId>
<artifactId>agent-plane-protocol</artifactId>
<version>1.12.17-SNAPSHOT</version>
<version>1.12.18-SNAPSHOT</version>
</dependency>
```

Expand All @@ -89,6 +90,7 @@ See [this sample configuration file](resources/dataplane.properties)

| Property | Required | Default/Example | Description | List |
|-----------------------------------------------|----------|--------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|------|
| cx.agent.matchmaking | | http://matchmaking-agent.internal | URL of the matchmaking agent (use internal one if null) | |
| cx.agent.asset.default | | urn:x-arq:DefaultGraph | IRI of the default graph (federated data catalogue) | |
| cx.agent.asset.file | | https://www.w3id.org/catenax/ontology,dataspace.ttl | Initial triples for the default graph (federated data catalogue) | L |
| cx.agent.accesspoint.name | | api | Matchmaking agent endpoint name (internal) | |
Expand All @@ -109,7 +111,7 @@ See [this sample configuration file](resources/dataplane.properties)
| cx.agent.service.deny | | ^$ | Regular expression for determining which IRIs are denied in SERVICE calls (on top level/federated data catalogue) | | |
| cx.agent.service.asset.allow | | (http&#124;edc)s://.* | Regular expression for determining which IRIs are allowed in delegated SERVICE calls (if not overriden by the cx-common:allowServicePattern address property) | |
| cx.agent.service.asset.deny | | ^$ | Regular expression for determining which IRIs are denied in delegated SERVICE calls (it not overridden by the cx-common:denyServicePattern address property) | | |
| cx.agent.dataspace.remotes | | http://consumer-edc-control:8282,http://tiera-edc-control:8282 | business partner control plane protocol urls to synchronize with | L |
| cx.agent.dataspace.remotes | | http://consumer-edc-control:8282,http://tiera-edc-control:8282 | business partner control plane protocol urls to synchronize with (if using internal matchmaking) | L |
| cx.agent.sparql.verbose | | false | Controls the verbosity of the SparQL Engine | |
| cx.agent.threadpool.size | | 4 | Number of threads pooled for any concurrent batch calls and synchronisation actions | |
| cx.agent.federation.batch.max | | 9223372036854775807 / 8 | Maximal number of tuples to send in one query | |
Expand Down
2 changes: 1 addition & 1 deletion agent-plane/agent-plane-protocol/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
<parent>
<groupId>org.eclipse.tractusx.agents.edc</groupId>
<artifactId>agent-plane</artifactId>
<version>1.12.17-SNAPSHOT</version>
<version>1.12.18-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
Loading

0 comments on commit 6917953

Please sign in to comment.