diff --git a/CHANGELOG.md b/CHANGELOG.md index 82018a3b3..9cc638b06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,28 @@ The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [released] +## [v1.4.0] - 14-12-2023 + +## Added +- Added script to automate the uploading of various passport types +- Added script to delete data from the data provider +- Added check for empty or null contractIds with retry attempts +- Added descriptive logs to search and create methods + +## Updated +- Updated ingress settings and backend configuration in the helm chart +- Refactored helm values to show only user relevant settings + +## Issued Fixed +- Fixed the timeout time for each negotiation +- Fixed the long waiting time by implementing timeout when doing the negotiation +- Fixed the null contract ids creation + +## Deleted +- Remove the legacy style to register/delete the testdata from the data provider + + ## [released] ## [v1.3.1] - 08-11-2023 diff --git a/charts/digital-product-pass/templates/deployment-backend.yaml b/charts/digital-product-pass/templates/deployment-backend.yaml index fe1208972..8be65d12f 100644 --- a/charts/digital-product-pass/templates/deployment-backend.yaml +++ b/charts/digital-product-pass/templates/deployment-backend.yaml @@ -79,8 +79,8 @@ spec: - name: backend-config mountPath: /app/config - name: pvc-backend - mountPath: /app/data - subPath: data + mountPath: /app/data/process + subPath: data/process - name: pvc-backend mountPath: /app/log subPath: log diff --git a/charts/digital-product-pass/values-beta.yaml b/charts/digital-product-pass/values-beta.yaml index a4617a433..4172b088c 100644 --- a/charts/digital-product-pass/values-beta.yaml +++ b/charts/digital-product-pass/values-beta.yaml @@ -96,4 +96,4 @@ oauth: clientId: clientSecret: realm: - appId: + appId: \ No newline at end of file diff --git a/charts/digital-product-pass/values-dev.yaml b/charts/digital-product-pass/values-dev.yaml index 55fdc43f1..04210dbd1 100644 --- a/charts/digital-product-pass/values-dev.yaml +++ b/charts/digital-product-pass/values-dev.yaml @@ -97,4 +97,4 @@ oauth: clientId: clientSecret: realm: - appId: + appId: \ No newline at end of file diff --git a/charts/digital-product-pass/values.yaml b/charts/digital-product-pass/values.yaml index 751c7ddb6..5e2e7d9ee 100644 --- a/charts/digital-product-pass/values.yaml +++ b/charts/digital-product-pass/values.yaml @@ -204,4 +204,4 @@ nodeSelector: {} tolerations: [] # [affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) to configure which nodes the pods can be scheduled on -affinity: {} +affinity: {} \ No newline at end of file diff --git a/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/config/DtrConfig.java b/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/config/DtrConfig.java index a23885c93..26ab220cb 100644 --- a/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/config/DtrConfig.java +++ b/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/config/DtrConfig.java @@ -152,6 +152,7 @@ public static class Timeouts{ Integer negotiation; Integer transfer; Integer digitalTwin; + Integer dtrRequestProcess; /** GETTERS AND SETTERS **/ public Integer getSearch() { @@ -179,6 +180,14 @@ public Integer getDigitalTwin() { public void setDigitalTwin(Integer digitalTwin) { this.digitalTwin = digitalTwin; } + + public Integer getDtrRequestProcess() { + return dtrRequestProcess; + } + + public void setDtrRequestProcess(Integer dtrRequestProcess) { + this.dtrRequestProcess = dtrRequestProcess; + } } /** diff --git a/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/http/controllers/api/ContractController.java b/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/http/controllers/api/ContractController.java index a553567f5..ad66e8036 100644 --- a/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/http/controllers/api/ContractController.java +++ b/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/http/controllers/api/ContractController.java @@ -155,6 +155,7 @@ public Response create(@Valid @RequestBody DiscoverySearch searchBody) { return httpUtil.buildResponse(response, httpResponse); } String processId = processManager.initProcess(); + LogUtil.printMessage("Creating process [" + processId + "] for "+searchBody.getType() + ": "+ searchBody.getId()); ConcurrentHashMap> dataModel = null; if(dtrConfig.getTemporaryStorage().getEnabled()) { try { @@ -187,7 +188,7 @@ public Response create(@Valid @RequestBody DiscoverySearch searchBody) { for(Dtr dtr: dtrs){ Long validUntil = dtr.getValidUntil(); - if(validUntil == null || validUntil < currentTimestamp){ + if(dtr.getContractId() == null || dtr.getContractId().isEmpty() || validUntil == null || validUntil < currentTimestamp){ requestDtrs = true; // If the cache invalidation time has come request Dtrs break; } @@ -270,7 +271,7 @@ public Response search(@Valid @RequestBody Search searchBody) { response = httpUtil.getBadRequest("No processId was found on the request body!"); return httpUtil.buildResponse(response, httpResponse); } - + String processId = searchBody.getProcessId(); if(processId.isEmpty()){ response = httpUtil.getBadRequest("Process id is required for decentral digital twin registry searches!"); @@ -286,9 +287,12 @@ public Response search(@Valid @RequestBody Search searchBody) { return httpUtil.buildResponse(response, httpResponse); } Boolean childrenCondition = searchBody.getChildren(); + String logPrint = "[" + processId + "] Creating search for "+searchBody.getIdType() + ": "+ searchBody.getId(); if(childrenCondition != null){ + LogUtil.printMessage(logPrint + " with drilldown enabled"); process = processManager.createProcess(processId, childrenCondition, httpRequest); // Store the children condition }else { + LogUtil.printMessage(logPrint + " with drilldown disabled"); process = processManager.createProcess(processId, httpRequest); } Status status = processManager.getStatus(processId); diff --git a/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/managers/DtrSearchManager.java b/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/managers/DtrSearchManager.java index 63f5154a9..999163b44 100644 --- a/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/managers/DtrSearchManager.java +++ b/consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/managers/DtrSearchManager.java @@ -69,7 +69,7 @@ public class DtrSearchManager { private ConcurrentHashMap> dtrDataModel; private ConcurrentHashMap catalogsCache; private final long searchTimeoutSeconds; - private final long negotiationTimeoutSeconds; + private final long dtrRequestProcessTimeout; private final String fileName = "dtrDataModel.json"; private String dtrDataModelFilePath; private State state; @@ -93,8 +93,7 @@ public DtrSearchManager(FileUtil fileUtil, JsonUtil jsonUtil, DataTransferServic this.dtrDataModelFilePath = this.createDataModelFile(); this.dtrDataModel = this.loadDtrDataModel(); this.searchTimeoutSeconds = this.dtrConfig.getTimeouts().getSearch(); - this.negotiationTimeoutSeconds = this.dtrConfig.getTimeouts().getNegotiation(); - + this.dtrRequestProcessTimeout = this.dtrConfig.getTimeouts().getDtrRequestProcess(); } /** GETTERS AND SETTERS **/ @@ -205,7 +204,7 @@ public void run() { } public void searchEndpoint(String processId, String bpn, String endpoint){ //Search Digital Twin Catalog for each connectionURL with a timeout time - Thread asyncThread = ThreadUtil.runThread(searchDigitalTwinCatalogExecutor(endpoint), "ProcessDtrDataModel"); + Thread asyncThread = ThreadUtil.runThread(searchDigitalTwinCatalogExecutor(endpoint), "SearchEndpoint"+processId+"-"+bpn+"-"+endpoint); try { if (!asyncThread.join(Duration.ofSeconds(searchTimeoutSeconds))) { asyncThread.interrupt(); @@ -228,11 +227,11 @@ public void searchEndpoint(String processId, String bpn, String endpoint){ if (contractOffers instanceof LinkedHashMap) { Dataset dataset = (Dataset) jsonUtil.bindObject(contractOffers, Dataset.class); if (dataset != null) { - Thread singleOfferThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr"); + Thread singleOfferThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr-"+processId+"-"+bpn+"-"+endpoint); try { - if (!singleOfferThread.join(Duration.ofSeconds(negotiationTimeoutSeconds))) { + if (!singleOfferThread.join(Duration.ofSeconds(this.dtrRequestProcessTimeout))) { singleOfferThread.interrupt(); - LogUtil.printWarning("Failed to retrieve the Catalog due a timeout for the URL: " + endpoint); + LogUtil.printWarning("Failed to retrieve do contract negotiations due a timeout for the URL: " + endpoint); return; } } catch (InterruptedException e) { @@ -246,11 +245,11 @@ public void searchEndpoint(String processId, String bpn, String endpoint){ return; } contractOfferList.parallelStream().forEach(dataset -> { - Thread multipleOffersThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr"); + Thread multipleOffersThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr-"+processId+"-"+bpn+"-"+endpoint); try { - if (!multipleOffersThread.join(Duration.ofSeconds(negotiationTimeoutSeconds))) { + if (!multipleOffersThread.join(Duration.ofSeconds(this.dtrRequestProcessTimeout))) { multipleOffersThread.interrupt(); - LogUtil.printWarning("Failed to retrieve the Catalog due a timeout for the URL: " + endpoint); + LogUtil.printWarning("Failed to retrieve the contract negotiations due a timeout for the URL: " + endpoint); } } catch (InterruptedException e) { throw new RuntimeException(e); @@ -426,16 +425,22 @@ private Runnable createAndSaveDtr(Dataset dataset, String bpn, String connection public void run() { try { Offer offer = dataTransferService.buildOffer(dataset, 0); - String builtDataEndpoint =CatenaXUtil.buildDataEndpoint(connectionUrl); + String builtDataEndpoint = CatenaXUtil.buildDataEndpoint(connectionUrl); IdResponse negotiationResponse = dataTransferService.doContractNegotiation(offer, bpn, builtDataEndpoint); if (negotiationResponse == null) { return; } - Negotiation negotiation = dataTransferService.seeNegotiation(negotiationResponse.getId()); + Integer millis = dtrConfig.getTimeouts().getNegotiation() * 1000; // Set max timeout from seconds to milliseconds + // If negotiation takes way too much time give timeout + Negotiation negotiation = ThreadUtil.timeout(millis, ()->dataTransferService.seeNegotiation(negotiationResponse.getId()), null); if (negotiation == null) { LogUtil.printWarning("It was not possible to do ContractNegotiation for URL: " + connectionUrl); return; } + if(negotiation.getContractAgreementId() == null || negotiation.getContractAgreementId().isEmpty()){ + LogUtil.printError("It was not possible to get an Contract Agreemment Id for the URL: " + connectionUrl); + return; + } Dtr dtr = new Dtr(negotiation.getContractAgreementId(), connectionUrl, offer.getAssetId(), bpn, DateTimeUtil.addHoursToCurrentTimestamp(dtrConfig.getTemporaryStorage().getLifetime())); if (dtrConfig.getTemporaryStorage().getEnabled()) { addConnectionToBpnEntry(bpn, dtr); diff --git a/consumer-backend/productpass/src/main/java/utils/ThreadUtil.java b/consumer-backend/productpass/src/main/java/utils/ThreadUtil.java index d92333315..da95667a0 100644 --- a/consumer-backend/productpass/src/main/java/utils/ThreadUtil.java +++ b/consumer-backend/productpass/src/main/java/utils/ThreadUtil.java @@ -134,12 +134,12 @@ public static V timeout(Integer milliseconds, Callable function, V timeou { try { ExecutorService executor = Executors.newSingleThreadExecutor(); - Future future = executor.submit(function); boolean timeout = false; V returnObject = null; try { + Future future = executor.submit(function); returnObject = future.get(milliseconds, TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { + } catch (Exception e) { timeout = true; } executor.shutdownNow(); diff --git a/consumer-backend/productpass/src/main/resources/application.yml b/consumer-backend/productpass/src/main/resources/application.yml index 7c2087be5..a0a524bbf 100644 --- a/consumer-backend/productpass/src/main/resources/application.yml +++ b/consumer-backend/productpass/src/main/resources/application.yml @@ -81,7 +81,8 @@ configuration: subModel: "/submodel-descriptors" timeouts: search: 10 - negotiation: 40 + negotiation: 10 + dtrRequestProcess: 40 transfer: 10 digitalTwin: 20 temporaryStorage: diff --git a/docs/RELEASE_USER.md b/docs/RELEASE_USER.md index c350cdba9..b867ce359 100644 --- a/docs/RELEASE_USER.md +++ b/docs/RELEASE_USER.md @@ -23,6 +23,16 @@ # Release Notes Digital Product Pass Application User friendly relase notes without especific technical details. +**November 14 2023 (Version 1.4.0)** +*14.12.2023* + +### Added +#### DPP test data uploader +A script is refactored to upload/remove testdata set from the data provider setup. This speeds up the automatic uploading of various passes types into the provider's digital twin registry, data service and EDC connector. + +### Updated +#### Optimize contract negotiation time +There was a long waiting time during the contract negotiation. This time is now reduced and the negotiation is perfomred faster. **November 08 2023 (Version 1.3.1)** *08.11.2023*