Merge pull request #148 from catenax-ng/bugfix/cmp-983/dtr-lookup

[ 3° ] - Release/v1.4.0 Bugfix/dtr lookup: Fixed bugs related to negotiation timeout and "gost" contract id
eclipse-tractusx · Dec 14, 2023 · ee78fa5 · ee78fa5
2 parents c16326c + c49b4d5
commit ee78fa5
Show file tree

Hide file tree

Showing 11 changed files with 73 additions and 22 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,28 @@
 
 The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [released]
+## [v1.4.0] - 14-12-2023
+
+## Added
+- Added script to automate the uploading of various passport types
+- Added script to delete data from the data provider
+- Added check for empty or null contractIds with retry attempts
+- Added descriptive logs to search and create methods
+
+## Updated
+- Updated ingress settings and backend configuration in the helm chart
+- Refactored helm values to show only user relevant settings
+
+## Issued Fixed
+- Fixed the timeout time for each negotiation
+- Fixed the long waiting time by implementing timeout when doing the negotiation
+- Fixed the null contract ids creation
+
+## Deleted
+- Remove the legacy style to register/delete the testdata from the data provider
+
+
 ## [released]
 ## [v1.3.1] - 08-11-2023
 

diff --git a/charts/digital-product-pass/templates/deployment-backend.yaml b/charts/digital-product-pass/templates/deployment-backend.yaml
@@ -79,8 +79,8 @@ spec:
             - name: backend-config
               mountPath: /app/config
             - name: pvc-backend
-              mountPath: /app/data
-              subPath: data
+              mountPath: /app/data/process
+              subPath: data/process
             - name: pvc-backend
               mountPath: /app/log
               subPath: log

diff --git a/charts/digital-product-pass/values-beta.yaml b/charts/digital-product-pass/values-beta.yaml
@@ -96,4 +96,4 @@ oauth:
     clientId: <path:material-pass/data/int/backend#clientId>
     clientSecret: <path:material-pass/data/int/backend#clientSecret>
   realm: <path:material-pass/data/int/oauth#realm>
-  appId: <path:material-pass/data/int/oauth#appId>
+  appId: <path:material-pass/data/int/oauth#appId>
diff --git a/charts/digital-product-pass/values-dev.yaml b/charts/digital-product-pass/values-dev.yaml
@@ -97,4 +97,4 @@ oauth:
     clientId: <path:material-pass/data/int/backend#clientId>
     clientSecret: <path:material-pass/data/int/backend#clientSecret>
   realm: <path:material-pass/data/int/oauth#realm>
-  appId: <path:material-pass/data/int/oauth#appId>
+  appId: <path:material-pass/data/int/oauth#appId>
diff --git a/charts/digital-product-pass/values.yaml b/charts/digital-product-pass/values.yaml
@@ -204,4 +204,4 @@ nodeSelector: {}
 tolerations: []
 
 # [affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) to configure which nodes the pods can be scheduled on
-affinity: {}
+affinity: {}
diff --git a/...-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/config/DtrConfig.java b/...-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/config/DtrConfig.java
@@ -152,6 +152,7 @@ public static class Timeouts{
         Integer negotiation;
         Integer transfer;
         Integer digitalTwin;
+        Integer dtrRequestProcess;
 
         /** GETTERS AND SETTERS **/
         public Integer getSearch() {
@@ -179,6 +180,14 @@ public Integer getDigitalTwin() {
         public void setDigitalTwin(Integer digitalTwin) {
             this.digitalTwin = digitalTwin;
         }
+
+        public Integer getDtrRequestProcess() {
+            return dtrRequestProcess;
+        }
+
+        public void setDtrRequestProcess(Integer dtrRequestProcess) {
+            this.dtrRequestProcess = dtrRequestProcess;
+        }
     }
 
     /**

diff --git a/...c/main/java/org/eclipse/tractusx/productpass/http/controllers/api/ContractController.java b/...c/main/java/org/eclipse/tractusx/productpass/http/controllers/api/ContractController.java
@@ -155,6 +155,7 @@ public Response create(@Valid @RequestBody DiscoverySearch searchBody) {
                 return httpUtil.buildResponse(response, httpResponse);
             }
             String processId = processManager.initProcess();
+            LogUtil.printMessage("Creating process [" + processId + "] for "+searchBody.getType() + ": "+ searchBody.getId());
             ConcurrentHashMap<String, List<Dtr>> dataModel = null;
             if(dtrConfig.getTemporaryStorage().getEnabled()) {
                 try {
@@ -187,7 +188,7 @@ public Response create(@Valid @RequestBody DiscoverySearch searchBody) {
                     for(Dtr dtr: dtrs){
 
                         Long validUntil  = dtr.getValidUntil();
-                        if(validUntil == null || validUntil < currentTimestamp){
+                        if(dtr.getContractId() == null || dtr.getContractId().isEmpty() || validUntil == null || validUntil < currentTimestamp){
                             requestDtrs = true; // If the cache invalidation time has come request Dtrs
                             break;
                         }
@@ -270,7 +271,7 @@ public Response search(@Valid @RequestBody Search searchBody) {
                 response = httpUtil.getBadRequest("No processId was found on the request body!");
                 return httpUtil.buildResponse(response, httpResponse);
             }
-
+            
             String processId = searchBody.getProcessId();
             if(processId.isEmpty()){
                 response = httpUtil.getBadRequest("Process id is required for decentral digital twin registry searches!");
@@ -286,9 +287,12 @@ public Response search(@Valid @RequestBody Search searchBody) {
                 return httpUtil.buildResponse(response, httpResponse);
             }
             Boolean childrenCondition = searchBody.getChildren();
+            String logPrint = "[" + processId + "] Creating search for "+searchBody.getIdType() + ": "+ searchBody.getId();
             if(childrenCondition != null){
+                LogUtil.printMessage(logPrint + " with drilldown enabled");
                 process = processManager.createProcess(processId, childrenCondition, httpRequest); // Store the children condition
             }else {
+                LogUtil.printMessage(logPrint + " with drilldown disabled");
                 process = processManager.createProcess(processId, httpRequest);
             }
             Status status = processManager.getStatus(processId);

diff --git a/...productpass/src/main/java/org/eclipse/tractusx/productpass/managers/DtrSearchManager.java b/...productpass/src/main/java/org/eclipse/tractusx/productpass/managers/DtrSearchManager.java
@@ -69,7 +69,7 @@ public class DtrSearchManager {
     private ConcurrentHashMap<String, List<Dtr>> dtrDataModel;
     private ConcurrentHashMap<String, Catalog> catalogsCache;
     private final long searchTimeoutSeconds;
-    private final long negotiationTimeoutSeconds;
+    private final long dtrRequestProcessTimeout;
     private final String fileName = "dtrDataModel.json";
     private String dtrDataModelFilePath;
     private State state;
@@ -93,8 +93,7 @@ public DtrSearchManager(FileUtil fileUtil, JsonUtil jsonUtil, DataTransferServic
         this.dtrDataModelFilePath = this.createDataModelFile();
         this.dtrDataModel = this.loadDtrDataModel();
         this.searchTimeoutSeconds = this.dtrConfig.getTimeouts().getSearch();
-        this.negotiationTimeoutSeconds = this.dtrConfig.getTimeouts().getNegotiation();
-
+        this.dtrRequestProcessTimeout = this.dtrConfig.getTimeouts().getDtrRequestProcess();
     }
 
     /** GETTERS AND SETTERS **/
@@ -205,7 +204,7 @@ public void run() {
     }
     public void searchEndpoint(String processId, String bpn, String endpoint){
         //Search Digital Twin Catalog for each connectionURL with a timeout time
-        Thread asyncThread = ThreadUtil.runThread(searchDigitalTwinCatalogExecutor(endpoint), "ProcessDtrDataModel");
+        Thread asyncThread = ThreadUtil.runThread(searchDigitalTwinCatalogExecutor(endpoint), "SearchEndpoint"+processId+"-"+bpn+"-"+endpoint);
         try {
             if (!asyncThread.join(Duration.ofSeconds(searchTimeoutSeconds))) {
                 asyncThread.interrupt();
@@ -228,11 +227,11 @@ public void searchEndpoint(String processId, String bpn, String endpoint){
         if (contractOffers instanceof LinkedHashMap) {
             Dataset dataset = (Dataset) jsonUtil.bindObject(contractOffers, Dataset.class);
             if (dataset != null) {
-                Thread singleOfferThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr");
+                Thread singleOfferThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr-"+processId+"-"+bpn+"-"+endpoint);
                 try {
-                    if (!singleOfferThread.join(Duration.ofSeconds(negotiationTimeoutSeconds))) {
+                    if (!singleOfferThread.join(Duration.ofSeconds(this.dtrRequestProcessTimeout))) {
                         singleOfferThread.interrupt();
-                        LogUtil.printWarning("Failed to retrieve the Catalog due a timeout for the URL: " + endpoint);
+                        LogUtil.printWarning("Failed to retrieve do contract negotiations due a timeout for the URL: " + endpoint);
                         return;
                     }
                 } catch (InterruptedException e) {
@@ -246,11 +245,11 @@ public void searchEndpoint(String processId, String bpn, String endpoint){
             return;
         }
         contractOfferList.parallelStream().forEach(dataset -> {
-            Thread multipleOffersThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr");
+            Thread multipleOffersThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr-"+processId+"-"+bpn+"-"+endpoint);
             try {
-                if (!multipleOffersThread.join(Duration.ofSeconds(negotiationTimeoutSeconds))) {
+                if (!multipleOffersThread.join(Duration.ofSeconds(this.dtrRequestProcessTimeout))) {
                     multipleOffersThread.interrupt();
-                    LogUtil.printWarning("Failed to retrieve the Catalog due a timeout for the URL: " + endpoint);
+                    LogUtil.printWarning("Failed to retrieve the contract negotiations due a timeout for the URL: " + endpoint);
                 }
             } catch (InterruptedException e) {
                 throw new RuntimeException(e);
@@ -426,16 +425,22 @@ private Runnable createAndSaveDtr(Dataset dataset, String bpn, String connection
             public void run() {
                 try {
                     Offer offer = dataTransferService.buildOffer(dataset, 0);
-                    String builtDataEndpoint =CatenaXUtil.buildDataEndpoint(connectionUrl);
+                    String builtDataEndpoint = CatenaXUtil.buildDataEndpoint(connectionUrl);
                     IdResponse negotiationResponse = dataTransferService.doContractNegotiation(offer, bpn, builtDataEndpoint);
                     if (negotiationResponse == null) {
                         return;
                     }
-                    Negotiation negotiation = dataTransferService.seeNegotiation(negotiationResponse.getId());
+                    Integer millis =  dtrConfig.getTimeouts().getNegotiation() * 1000; // Set max timeout from seconds to milliseconds
+                    // If negotiation takes way too much time give timeout
+                    Negotiation negotiation = ThreadUtil.timeout(millis, ()->dataTransferService.seeNegotiation(negotiationResponse.getId()), null);
                     if (negotiation == null) {
                         LogUtil.printWarning("It was not possible to do ContractNegotiation for URL: " + connectionUrl);
                         return;
                     }
+                    if(negotiation.getContractAgreementId() == null || negotiation.getContractAgreementId().isEmpty()){
+                        LogUtil.printError("It was not possible to get an Contract Agreemment Id for the URL: " + connectionUrl);
+                        return;
+                    }
                     Dtr dtr = new Dtr(negotiation.getContractAgreementId(), connectionUrl, offer.getAssetId(), bpn, DateTimeUtil.addHoursToCurrentTimestamp(dtrConfig.getTemporaryStorage().getLifetime()));
                     if (dtrConfig.getTemporaryStorage().getEnabled()) {
                         addConnectionToBpnEntry(bpn, dtr);

diff --git a/consumer-backend/productpass/src/main/java/utils/ThreadUtil.java b/consumer-backend/productpass/src/main/java/utils/ThreadUtil.java
@@ -134,12 +134,12 @@ public static <V> V timeout(Integer milliseconds, Callable<V> function, V timeou
     {
         try {
             ExecutorService executor = Executors.newSingleThreadExecutor();
-            Future<V> future = executor.submit(function);
             boolean timeout = false;
             V returnObject = null;
             try {
+                Future<V> future = executor.submit(function);
                 returnObject = future.get(milliseconds, TimeUnit.MILLISECONDS);
-            } catch (TimeoutException e) {
+            } catch (Exception e) {
                 timeout = true;
             }
             executor.shutdownNow();

diff --git a/consumer-backend/productpass/src/main/resources/application.yml b/consumer-backend/productpass/src/main/resources/application.yml
@@ -81,7 +81,8 @@ configuration:
       subModel: "/submodel-descriptors"
     timeouts:
       search: 10
-      negotiation: 40
+      negotiation: 10
+      dtrRequestProcess: 40
       transfer: 10
       digitalTwin: 20
     temporaryStorage:

diff --git a/docs/RELEASE_USER.md b/docs/RELEASE_USER.md
@@ -23,6 +23,16 @@
 # Release Notes Digital Product Pass Application
 User friendly relase notes without especific technical details.
 
+**November 14 2023 (Version 1.4.0)**
+*14.12.2023*
+
+### Added
+#### DPP test data uploader
+A script is refactored to upload/remove testdata set from the data provider setup. This speeds up the automatic uploading of various passes types into the provider's digital twin registry, data service and EDC connector.
+
+### Updated
+#### Optimize contract negotiation time
+There was a long waiting time during the contract negotiation. This time is now reduced and the negotiation is perfomred faster.
 
 **November 08 2023 (Version 1.3.1)**
 *08.11.2023*