Saas installation

.all-contributorsrc

@@ -100,6 +100,16 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "98jan",
+      "name": "Jan",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46779261?v=4",
+      "profile": "https://github.com/98jan",
+      "contributions": [
+        "code",
+        "infra"
+      ]
     }
   ]
 }

.github/workflows/backend-sonarqube-analysis.yml

@@ -5,12 +5,6 @@ on:
     paths:
       - backend/**
       - .github/workflows/backend-sonarqube-analysis.yml
-    branches: [ "main" ]
-  pull_request:
-    paths:
-      - backend/**
-      - .github/workflows/backend-sonarqube-analysis.yml
-    types: [ opened, reopened, synchronize ]
 
 jobs:
   build-and-analyse:
@@ -25,10 +19,13 @@ jobs:
             label: "AZ Document AI"
           - key: "hopps-app_hopps-vereine"
             folder: "app.hopps.org"
-            label: "Vereine"
+            label: "Org"
           - key: "hopps-app_hopps-fin"
             folder: "app.hopps.fin"
             label: "Finanzen"
+          - key: "hopps-app_zugferd"
+            folder: "app.hopps.zugferd"
+            label: "ZUGFeRD"
 
     steps:
       - uses: actions/checkout@v4
@@ -39,8 +36,14 @@ jobs:
           distribution: "temurin"
           cache: maven
 
+      - name: mvn install parent
+        run: mvn -B -N clean install --file backend/pom.xml
+
+      - name: mvn install commons
+        run: mvn -B install --file backend/app.hopps.commons/pom.xml
+
       - name: mvn package
-        run: mvn -B verify --file backend/pom.xml
+        run: mvn -B verify --file backend/${{ matrix.sonar.folder }}/pom.xml
 
       - name: Cache SonarQube packages
         uses: actions/cache@v4

.github/workflows/backend.yml

@@ -29,6 +29,8 @@ jobs:
             directory: app.hopps.org
           - label: az-document-ai
             directory: app.hopps.az-document-ai
+          - label: zugferd
+            directory: app.hopps.zugferd
 
     runs-on: ubuntu-latest
     steps:
@@ -40,9 +42,15 @@ jobs:
           distribution: 'temurin'
           cache: maven
 
+      - name: mvn install parent
+        run: mvn -B -N clean install --file backend/pom.xml
+
+      - name: mvn install commons
+        run: mvn -B install --file backend/app.hopps.commons/pom.xml
+
       # only build relevant service
       - name: mvn package
-        run: mvn -B package --file backend/${{ matrix.service.directory }}/pom.xml
+        run: mvn -B clean package --file backend/${{ matrix.service.directory }}/pom.xml
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta

.github/workflows/dependency-track-analysis.yml

@@ -5,12 +5,6 @@ on:
     paths:
       - backend/**
       - .github/workflows/dependency-track-analysis.yml
-    branches: [ "main" ]
-  pull_request:
-    paths:
-      - backend/**
-      - .github/workflows/dependency-track-analysis.yml
-    types: [ opened, reopened, synchronize ]
 
 jobs:
   upload-bom:
@@ -19,7 +13,11 @@ jobs:
 
     strategy:
       matrix:
-        projectName: [ "app.hopps.org", "app.hopps.az-document-ai", "app.hopps.fin" ]
+        projectName:
+          - "app.hopps.org"
+          - "app.hopps.az-document-ai"
+          - "app.hopps.fin"
+          - "app.hopps.zugferd"
 
     steps:
       # Checkout the code
@@ -32,7 +30,7 @@ jobs:
           cache: maven
 
       - name: mvn makeAggregateBom
-        run: mvn org.cyclonedx:cyclonedx-maven-plugin:2.8.0:makeBom --file backend/pom.xml
+        run: mvn org.cyclonedx:cyclonedx-maven-plugin:2.8.0:makeBom --file backend/${{ matrix.projectName }}/pom.xml
 
       - name: Get current project version
         uses: avides/[email protected]

.gitignore

@@ -193,3 +193,10 @@ $RECYCLE.BIN/
 
 # Drawio
 *.drawio.bkp
+
+# helm
+./charts/hopps/charts
+
+# unencrypted secrets
+*decrypted.env
+age.agekey

README.md

@@ -51,6 +51,7 @@ unter [hub.docker.com/r/apache/incubator-kie-kogito-data-index-ephemeral/tags](h
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/eliashehme"><img src="https://avatars.githubusercontent.com/u/176706065?v=4?s=100" width="100px;" alt="Elias Hehme"/><br /><sub><b>Elias Hehme</b></sub></a><br /><a href="#projectManagement-eliashehme" title="Project Management">📆</a> <a href="#business-eliashehme" title="Business development">💼</a></td>
       <td align="center" valign="top" width="14.28%"><a href="http://www.nachhilfe-paf.de"><img src="https://avatars.githubusercontent.com/u/119429802?v=4?s=100" width="100px;" alt="Simon Krumböck"/><br /><sub><b>Simon Krumböck</b></sub></a><br /><a href="#projectManagement-sk2675" title="Project Management">📆</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/3thr3n"><img src="https://avatars.githubusercontent.com/u/33932819?v=4?s=100" width="100px;" alt="3thr3n"/><br /><sub><b>3thr3n</b></sub></a><br /><a href="#code-3thr3n" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/98jan"><img src="https://avatars.githubusercontent.com/u/46779261?v=4?s=100" width="100px;" alt="Jan"/><br /><sub><b>Jan</b></sub></a><br /><a href="#code-98jan" title="Code">💻</a> <a href="#infra-98jan" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
     </tr>
   </tbody>
 </table>

backend/app.hopps.az-document-ai/pom.xml

@@ -23,7 +23,7 @@
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
         <quarkus.platform.group-id>io.quarkus.platform</quarkus.platform.group-id>
-        <quarkus.platform.version>3.16.4</quarkus.platform.version>
+        <quarkus.platform.version>3.17.6</quarkus.platform.version>
         <skipITs>true</skipITs>
         <surefire-plugin.version>3.5.2</surefire-plugin.version>
         <sonar.coverage.jacoco.xmlReportPaths>
@@ -44,6 +44,11 @@
     </dependencyManagement>
 
     <dependencies>
+        <dependency>
+            <groupId>app.hopps</groupId>
+            <artifactId>commons</artifactId>
+            <version>1.0.0-SNAPSHOT</version>
+        </dependency>
         <!-- quarkus -->
         <dependency>
             <groupId>io.quarkus</groupId>
@@ -69,6 +74,22 @@
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-smallrye-health</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-micrometer</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-micrometer-registry-prometheus</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-oidc-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-rest-client-jackson</artifactId>
+        </dependency>
         <!-- third party -->
         <dependency>
             <groupId>com.azure</groupId>

backend/app.hopps.az-document-ai/src/main/java/app/hopps/AzureAiService.java

@@ -1,73 +1,98 @@
 package app.hopps;
 
-import app.hopps.model.InvoiceData;
-import app.hopps.model.ReceiptData;
-import com.azure.ai.documentintelligence.DocumentIntelligenceClient;
-import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder;
-import com.azure.ai.documentintelligence.models.AnalyzeDocumentRequest;
+import app.hopps.auth.FinRestClientImpl;
+import app.hopps.commons.DocumentData;
+import app.hopps.commons.InvoiceData;
+import app.hopps.commons.ReceiptData;
+import app.hopps.model.InvoiceDataHelper;
+import app.hopps.model.ReceiptDataHelper;
 import com.azure.ai.documentintelligence.models.AnalyzeResult;
-import com.azure.ai.documentintelligence.models.AnalyzeResultOperation;
 import com.azure.ai.documentintelligence.models.Document;
-import com.azure.core.credential.AzureKeyCredential;
-import com.azure.core.util.polling.SyncPoller;
+import io.quarkus.oidc.client.OidcClient;
+import io.quarkus.oidc.client.runtime.TokensHelper;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.net.URISyntaxException;
 import java.net.URL;
+import java.time.Duration;
 import java.util.List;
 
 @ApplicationScoped
 public class AzureAiService {
 
-    private final Logger LOGGER = LoggerFactory.getLogger(AzureAiService.class);
+    private static final Logger LOG = LoggerFactory.getLogger(AzureAiService.class);
+
+    private final AzureDocumentConnector azureDocumentConnector;
 
     @ConfigProperty(name = "app.hopps.az-document-ai.azure.invoiceModelId")
     String invoiceModelId;
 
     @ConfigProperty(name = "app.hopps.az-document-ai.azure.receiptModelId")
     String receiptModelId;
 
+    private final OidcClient oidcClient;
+    private final TokensHelper tokensHelper = new TokensHelper();
+
     @Inject
-    AzureDocumentConnector azureDocumentConnector;
+    public AzureAiService(AzureDocumentConnector azureDocumentConnector, OidcClient oidcClient) {
+        this.azureDocumentConnector = azureDocumentConnector;
+        this.oidcClient = oidcClient;
+    }
 
-    public ReceiptData scanReceipt(URL imageUrl) {
-        Document document = scanDocument(receiptModelId, imageUrl);
+    public ReceiptData scanReceipt(DocumentData documentData) {
+        Document document = scanDocument(receiptModelId, documentData.internalFinUrl());
         if (document == null) {
             return null;
         }
 
-        return ReceiptData.fromDocument(document);
+        return ReceiptDataHelper.fromDocument(documentData.referenceKey(), document);
     }
 
-    public InvoiceData scanInvoice(URL imageUrl) {
-        Document document = scanDocument(invoiceModelId, imageUrl);
+    public InvoiceData scanInvoice(DocumentData documentData) {
+        Document document = scanDocument(invoiceModelId, documentData.internalFinUrl());
         if (document == null) {
             return null;
         }
 
-        LOGGER.info("Scanned document: {}", document.getFields());
+        LOG.info("Scanned document: {}", document.getFields());
 
-        return InvoiceData.fromDocument(document);
+        return InvoiceDataHelper.fromDocument(documentData.referenceKey(), document);
     }
 
-    private Document scanDocument(String modelId, URL imageUrl) {
-        LOGGER.info("(model={}) Starting scan of document: '{}'", modelId, imageUrl);
+    private Document scanDocument(String modelId, URL documentUrl) {
+        LOG.info("(model={}) Starting scan of document: '{}'", modelId, documentUrl);
+        byte[] documentBytes = fetchDocument(documentUrl);
 
-        AnalyzeResult analyzeLayoutResult = azureDocumentConnector.getAnalyzeResult(modelId, imageUrl);
+        AnalyzeResult analyzeLayoutResult = azureDocumentConnector.getAnalyzeResult(modelId, documentBytes);
         List<Document> documents = analyzeLayoutResult.getDocuments();
 
         if (documents.isEmpty()) {
-            LOGGER.error("Couldn't analyze document '{}'", imageUrl);
+            LOG.error("Couldn't analyze document '{}'", documentUrl);
             return null;
         } else if (documents.size() > 1) {
-            LOGGER.warn("Document analysis found {} documents, using first one", documents.size());
+            LOG.warn("Document analysis found {} documents, using first one", documents.size());
         }
 
-        LOGGER.info("(model={}) Scan successfully completed for: '{}'", modelId, imageUrl);
+        LOG.info("(model={}) Scan successfully completed for: '{}'", modelId, documentUrl);
 
         return documents.getFirst();
     }
+
+    private byte[] fetchDocument(URL documentUrl) {
+        try {
+            FinRestClientImpl finRestClient = new FinRestClientImpl(documentUrl);
+            String accessToken = tokensHelper.getTokens(oidcClient)
+                    .await()
+                    .atMost(Duration.ofSeconds(3))
+                    .getAccessToken();
+
+            return finRestClient.getDocumentBase64(accessToken);
+        } catch (URISyntaxException e) {
+            throw new IllegalStateException("Couldn't fetch document", e);
+        }
+    }
 }

backend/app.hopps.az-document-ai/src/main/java/app/hopps/AzureDocumentConnector.java

@@ -10,8 +10,6 @@
 import jakarta.enterprise.context.ApplicationScoped;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 
-import java.net.URL;
-
 @ApplicationScoped
 public class AzureDocumentConnector {
 
@@ -26,7 +24,7 @@ public AzureDocumentConnector(
                 .buildClient();
     }
 
-    public AnalyzeResult getAnalyzeResult(String modelId, URL imageUrl) {
+    public AnalyzeResult getAnalyzeResult(String modelId, byte[] imageBytes) {
         SyncPoller<AnalyzeResultOperation, AnalyzeResult> analyzeLayoutPoller = azureClient.beginAnalyzeDocument(
                 modelId,
                 null,
@@ -36,7 +34,7 @@ public AnalyzeResult getAnalyzeResult(String modelId, URL imageUrl) {
                 null,
                 null,
                 null,
-                new AnalyzeDocumentRequest().setUrlSource(imageUrl.toString()));
+                new AnalyzeDocumentRequest().setBase64Source(imageBytes));
 
         return analyzeLayoutPoller.getFinalResult();
     }

backend/app.hopps.az-document-ai/src/main/java/app/hopps/DevDocumentConnector.java

@@ -1,37 +1,42 @@
 package app.hopps;
 
-import app.hopps.model.InvoiceData;
-import app.hopps.model.ReceiptData;
+import app.hopps.commons.DocumentData;
+import app.hopps.commons.DocumentType;
+import app.hopps.commons.InvoiceData;
+import app.hopps.commons.ReceiptData;
 import io.quarkus.arc.profile.IfBuildProfile;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
-import jakarta.ws.rs.GET;
 import jakarta.ws.rs.POST;
 import jakarta.ws.rs.Path;
 
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.net.URL;
 
 @ApplicationScoped
 @IfBuildProfile("dev")
 @Path("/documents/scan")
 public class DevDocumentConnector {
+    private final AzureAiService aiService;
 
     @Inject
-    AzureAiService aiService;
+    public DevDocumentConnector(AzureAiService aiService) {
+        this.aiService = aiService;
+    }
 
     @Path("/invoice")
     @POST
     public InvoiceData scanInvoice(String imageUrl) throws URISyntaxException, MalformedURLException {
-        return aiService.scanInvoice(new URI(imageUrl).toURL());
+        DocumentData documentData = new DocumentData(new URI(imageUrl).toURL(), -1L, DocumentType.INVOICE);
+        return aiService.scanInvoice(documentData);
     }
 
     @Path("/receipt")
     @POST
     public ReceiptData scanReceipt(String imageUrl) throws URISyntaxException, MalformedURLException {
-        return aiService.scanReceipt(new URI(imageUrl).toURL());
+        DocumentData documentData = new DocumentData(new URI(imageUrl).toURL(), -1L, DocumentType.RECEIPT);
+        return aiService.scanReceipt(documentData);
     }
 
 }