From f6ebdbc5a59bcbdcd4bdda7ac84405b4f6ca0e9b Mon Sep 17 00:00:00 2001 From: Ivo Leist Date: Mon, 11 Dec 2023 20:18:22 +0100 Subject: [PATCH 1/3] added an env variable for the filepond timeout + narrowed down what is the cause for the file upload retrigger after the deletion --- convertPheno_client/index.html | 1 + convertPheno_client/package.json | 1 - convertPheno_client/pnpm-lock.yaml | 129 +----------------- convertPheno_client/replace_env_vars.sh | 4 +- .../inputFilesPond/InputFilesPond.jsx | 40 +++++- docker-compose.dev.yml | 1 + docker-compose.prod.yml | 5 +- docker-compose.yml | 1 + makefile | 4 +- nginx_mountpoint/replace_env_vars.sh | 1 + 10 files changed, 46 insertions(+), 141 deletions(-) diff --git a/convertPheno_client/index.html b/convertPheno_client/index.html index fecee9d8..ffa4f443 100644 --- a/convertPheno_client/index.html +++ b/convertPheno_client/index.html @@ -20,6 +20,7 @@ window.REACT_APP_SECURITY = "{{VITE_SECURITY}}"; window.REACT_APP_API_URL = "{{VITE_API_URL}}"; window.REACT_APP_KC_CONFIG = "{{VITE_KC_CONFIG}}"; + window.REACT_APP_FILEPOND_TIMEOUT = "{{VITE_FILEPOND_TIMEOUT}}"; window.REACT_APP_MATOMO_URL = "{{MATOMO_TAG_MANAGER_URL}}"; diff --git a/convertPheno_client/package.json b/convertPheno_client/package.json index 97e8cb78..2754e650 100644 --- a/convertPheno_client/package.json +++ b/convertPheno_client/package.json @@ -22,7 +22,6 @@ "filepond-plugin-file-validate-type": "^1.2.8", "keycloak-js": "^23.0.1", "react": "^18.2.0", - "react-bootstrap": "^2.9.1", "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^16.0.1", "react-dom": "^18.2.0", diff --git a/convertPheno_client/pnpm-lock.yaml b/convertPheno_client/pnpm-lock.yaml index f9728da5..b0556487 100644 --- a/convertPheno_client/pnpm-lock.yaml +++ b/convertPheno_client/pnpm-lock.yaml @@ -59,9 +59,6 @@ dependencies: react: specifier: ^18.2.0 version: 18.2.0 - react-bootstrap: - specifier: ^2.9.1 - version: 2.9.1(@types/react@18.2.5)(react-dom@18.2.0)(react@18.2.0) react-dnd: specifier: ^16.0.1 version: 16.0.1(@types/node@18.15.10)(@types/react@18.2.5)(react@18.2.0) @@ -4382,15 +4379,6 @@ packages: resolution: {integrity: sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==} dev: false - /@react-aria/ssr@3.6.0(react@18.2.0): - resolution: {integrity: sha512-OFiYQdv+Yk7AO7IsQu/fAEPijbeTwrrEYvdNoJ3sblBBedD5j5fBTNWrUPNVlwC4XWWnWTCMaRIVsJujsFiWXg==} - peerDependencies: - react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 - dependencies: - '@swc/helpers': 0.4.14 - react: 18.2.0 - dev: false - /@react-dnd/asap@5.0.2: resolution: {integrity: sha512-WLyfoHvxhs0V9U+GTsGilGgf2QsPl6ZZ44fnv0/b8T3nQyvzxidxsg/ZltbWssbsRDlYW8UKSQMTGotuTotZ6A==} dev: false @@ -4407,34 +4395,6 @@ packages: resolution: {integrity: sha512-so+DHzZKsoOcoXrILB4rqDkMDy7NLMErRdOxvzvOKb507YINKUP4Di+shbTZDhSE/pBZ+vr7XGIpcOO0VLSA+Q==} engines: {node: '>=14.0.0'} - /@restart/hooks@0.4.9(react@18.2.0): - resolution: {integrity: sha512-3BekqcwB6Umeya+16XPooARn4qEPW6vNvwYnlofIYe6h9qG1/VeD7UvShCWx11eFz5ELYmwIEshz+MkPX3wjcQ==} - peerDependencies: - react: '>=16.8.0' - dependencies: - dequal: 2.0.3 - react: 18.2.0 - dev: false - - /@restart/ui@1.6.6(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-eC3puKuWE1SRYbojWHXnvCNHGgf3uzHCb6JOhnF4OXPibOIPEkR1sqDSkL643ydigxwh+ruCa1CmYHlzk7ikKA==} - peerDependencies: - react: '>=16.14.0' - react-dom: '>=16.14.0' - dependencies: - '@babel/runtime': 7.23.5 - '@popperjs/core': 2.11.8 - '@react-aria/ssr': 3.6.0(react@18.2.0) - '@restart/hooks': 0.4.9(react@18.2.0) - '@types/warning': 3.0.0 - dequal: 2.0.3 - dom-helpers: 5.2.1 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - uncontrollable: 8.0.4(react@18.2.0) - warning: 4.0.3 - dev: false - /@rollup/pluginutils@5.0.2: resolution: {integrity: sha512-pTd9rIsP92h+B6wWwFbW8RkZv4hiR/xKsqre4SIuAOaOEQRxi0lqLke9k2/7WegC85GgUs9pjmOjCUi3In4vwA==} engines: {node: '>=14.0.0'} @@ -5854,12 +5814,6 @@ packages: file-system-cache: 2.1.1 dev: true - /@swc/helpers@0.4.14: - resolution: {integrity: sha512-4C7nX/dvpzB7za4Ql9K81xK3HPxCpHMgwTZVyf+9JQ6VUbn9jjZVN7/Nkdz/Ugzs2CSjqnL/UPXroiVBVHUWUw==} - dependencies: - tslib: 2.5.0 - dev: false - /@tanstack/query-core@4.36.1: resolution: {integrity: sha512-DJSilV5+ytBP1FbFcEJovv4rnnm/CokuVvrBEtW/Va9DvuJ3HksbXUJEpI0aV1KtuL4ZoO9AVE6PyNLzF7tLeA==} @@ -6225,10 +6179,6 @@ packages: resolution: {integrity: sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==} dev: true - /@types/warning@3.0.0: - resolution: {integrity: sha512-t/Tvs5qR47OLOr+4E9ckN8AmP2Tf16gWq+/qA4iUGS/OOyHVO8wv2vjJuX8SNOUTJyWb+2t7wJm6cXILFnOROA==} - dev: false - /@types/webpack-env@1.18.1: resolution: {integrity: sha512-D0HJET2/UY6k9L6y3f5BL+IDxZmPkYmPT4+qBrRdmRLYRuV0qNKizMgTvYxXZYn+36zjPeoDZAEYBCM6XB+gww==} dev: true @@ -7530,10 +7480,6 @@ packages: engines: {node: '>=8'} dev: true - /classnames@2.3.2: - resolution: {integrity: sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==} - dev: false - /clean-css@5.3.2: resolution: {integrity: sha512-JVJbM+f3d3Q704rF4bqQ5UUyTtuJ0JRKNbTKVEeujCCBoMdkEi+V+e8oktO9qGQNSvHrFTM6JZRXrUvGR1czww==} engines: {node: '>= 10.0'} @@ -8207,6 +8153,7 @@ packages: /dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} + dev: true /destroy@1.2.0: resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} @@ -10056,12 +10003,6 @@ packages: engines: {node: '>= 0.10'} dev: true - /invariant@2.2.4: - resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} - dependencies: - loose-envify: 1.4.0 - dev: false - /ip@2.0.0: resolution: {integrity: sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==} dev: true @@ -12553,16 +12494,6 @@ packages: sisteransi: 1.0.5 dev: true - /prop-types-extra@1.1.1(react@18.2.0): - resolution: {integrity: sha512-59+AHNnHYCdiC+vMwY52WmvP5dM3QLeoumYuEyceQDi9aEhtwN9zIQ2ZNo25sMyXnbh32h+P1ezDsUpUH3JAew==} - peerDependencies: - react: '>=0.14.0' - dependencies: - react: 18.2.0 - react-is: 16.13.1 - warning: 4.0.3 - dev: false - /prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} dependencies: @@ -12697,33 +12628,6 @@ packages: unpipe: 1.0.0 dev: true - /react-bootstrap@2.9.1(@types/react@18.2.5)(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-ezgmh/ARCYp18LbZEqPp0ppvy+ytCmycDORqc8vXSKYV3cer4VH7OReV8uMOoKXmYzivJTxgzGHalGrHamryHA==} - peerDependencies: - '@types/react': '>=16.14.8' - react: '>=16.14.0' - react-dom: '>=16.14.0' - peerDependenciesMeta: - '@types/react': - optional: true - dependencies: - '@babel/runtime': 7.23.5 - '@restart/hooks': 0.4.9(react@18.2.0) - '@restart/ui': 1.6.6(react-dom@18.2.0)(react@18.2.0) - '@types/react': 18.2.5 - '@types/react-transition-group': 4.4.10 - classnames: 2.3.2 - dom-helpers: 5.2.1 - invariant: 2.2.4 - prop-types: 15.8.1 - prop-types-extra: 1.1.1(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - react-transition-group: 4.4.5(react-dom@18.2.0)(react@18.2.0) - uncontrollable: 7.2.1(react@18.2.0) - warning: 4.0.3 - dev: false - /react-colorful@5.6.1(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-1exovf0uGTGyq5mXQT0zgQ80uvj2PCwvF8zY1RN9/vbJVSjSo3fsB/4L3ObbF7u70NduSiK4xu4Y6q1MHoUGEw==} peerDependencies: @@ -12869,10 +12773,6 @@ packages: resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==} dev: false - /react-lifecycles-compat@3.0.4: - resolution: {integrity: sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA==} - dev: false - /react-refresh@0.11.0: resolution: {integrity: sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==} engines: {node: '>=0.10.0'} @@ -14189,6 +14089,7 @@ packages: /tslib@2.5.0: resolution: {integrity: sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==} + dev: true /tsutils@3.21.0(typescript@4.9.5): resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} @@ -14321,26 +14222,6 @@ packages: through: 2.3.8 dev: true - /uncontrollable@7.2.1(react@18.2.0): - resolution: {integrity: sha512-svtcfoTADIB0nT9nltgjujTi7BzVmwjZClOmskKu/E8FW9BXzg9os8OLr4f8Dlnk0rYWJIWr4wv9eKUXiQvQwQ==} - peerDependencies: - react: '>=15.0.0' - dependencies: - '@babel/runtime': 7.23.5 - '@types/react': 18.2.5 - invariant: 2.2.4 - react: 18.2.0 - react-lifecycles-compat: 3.0.4 - dev: false - - /uncontrollable@8.0.4(react@18.2.0): - resolution: {integrity: sha512-ulRWYWHvscPFc0QQXvyJjY6LIXU56f0h8pQFvhxiKk5V1fcI8gp9Ht9leVAhrVjzqMw0BgjspBINx9r6oyJUvQ==} - peerDependencies: - react: '>=16.14.0' - dependencies: - react: 18.2.0 - dev: false - /unfetch@4.2.0: resolution: {integrity: sha512-F9p7yYCn6cIW9El1zi0HI6vqpeIvBsr3dSuRO6Xuppb1u5rXpCPmMvLSyECLhybr9isec8Ohl0hPekMVrEinDA==} dev: true @@ -14663,12 +14544,6 @@ packages: makeerror: 1.0.12 dev: true - /warning@4.0.3: - resolution: {integrity: sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w==} - dependencies: - loose-envify: 1.4.0 - dev: false - /watchpack@2.4.0: resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==} engines: {node: '>=10.13.0'} diff --git a/convertPheno_client/replace_env_vars.sh b/convertPheno_client/replace_env_vars.sh index 7ddd6abc..46d4695c 100755 --- a/convertPheno_client/replace_env_vars.sh +++ b/convertPheno_client/replace_env_vars.sh @@ -18,8 +18,8 @@ echo $MATOMO_TAG_MANAGER_URL echo "=== DEBUG: End of check if env variables are set ===" # Define the placeholders and their corresponding environment variables -PLACEHOLDERS="{{VITE_SECURITY}} {{VITE_API_URL}} {{VITE_KC_CONFIG}} {{MATOMO_TAG_MANAGER_URL}}" -ENV_VARS="VITE_SECURITY VITE_API_URL VITE_KC_CONFIG MATOMO_TAG_MANAGER_URL" +PLACEHOLDERS="{{VITE_SECURITY}} {{VITE_API_URL}} {{VITE_KC_CONFIG}} {{VITE_FILEPOND_TIMEOUT}} {{MATOMO_TAG_MANAGER_URL}}" +ENV_VARS="VITE_SECURITY VITE_API_URL VITE_KC_CONFIG VITE_FILEPOND_TIMEOUT MATOMO_TAG_MANAGER_URL" # Replace the placeholders with the environment variable values for placeholder in $PLACEHOLDERS; do diff --git a/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx b/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx index b224df6e..89b996d4 100644 --- a/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx +++ b/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx @@ -26,7 +26,7 @@ import { } from "@mui/material"; import InfoIcon from "@mui/icons-material/Info"; -import {toast} from "react-toastify"; +import { toast } from "react-toastify"; import { FilePond, registerPlugin } from "react-filepond"; import "filepond/dist/filepond.min.css"; @@ -41,6 +41,15 @@ const api_endpoint = ? window.REACT_APP_API_URL : import.meta.env.VITE_API_URL; +console.log("api_endpoint", api_endpoint); + +const filepondTimeout = + process.env.NODE_ENV === "production" + ? parseInt(window.REACT_APP_FILEPOND_TIMEOUT) + : parseInt(import.meta.env.VITE_FILEPOND_TIMEOUT); + +console.log("filepondTimeout", filepondTimeout); + function CustomAlert(props) { const { info } = props; @@ -84,7 +93,7 @@ export default function InputFilesPond(props) { setFilesUploadFinished, setRunExampleData, inputFormat, - uploadedFiles + uploadedFiles, } = props; const [files, setFiles] = useState([]); @@ -105,11 +114,11 @@ export default function InputFilesPond(props) { */ const returnedFileName = JSON.parse(file.serverId).tempFilename; - const {filename, fileExtension} = file; + const { filename, fileExtension } = file; if (filename in uploadedFiles) { toast.error("File already uploaded"); - file.setMetadata("processingAborted", true) + file.setMetadata("processingAborted", true); file.abortProcessing(); return; } @@ -245,16 +254,16 @@ export default function InputFilesPond(props) { onupdatefiles={setFiles} allowMultiple={allowMultipleMapping[inputFormat]} maxFiles={getFileUploadInfo(inputFormat).fileCount} - server={{ url: `${api_endpoint}api/submission/upload`, - timeout: 7000, // 7 seconds + timeout: filepondTimeout, process: { headers: { Authorization: auth.getToken(), "X-Custom-InputFormat": inputFormat, }, onerror: (response) => { + console.log("error", response); errorHandling(response); }, }, @@ -273,10 +282,27 @@ export default function InputFilesPond(props) { setFilesUploadFinished(false); const fileName = file.filename; + console.log("files", files); + // only update state if the file processing was finished (status 2 = IDLE) // for reference see // github.com/pqina/filepond-docs/blob/master/content/patterns/API/filepond-object.md#filestatus-enum - if (file.status === 2 && file.getMetadata("processingAborted") !== true) { + + // TODO + // below is not working on the production environment + // It will always retrigger the re-upload of the file + + // Potential solution: + // do not have a setUploadedFile call here + // but rather use the files set by FilePond + // either use this state directly + // or use a useEffect hook to update the uploadedFiles state + // whenever the files state changes + + if ( + file.status === 2 && + file.getMetadata("processingAborted") !== true + ) { setUploadedFiles((prev) => { const prevCopy = { ...prev }; delete prevCopy[fileName]; diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 230bbb76..2e5df3b8 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -77,6 +77,7 @@ services: VITE_SECURITY: true VITE_API_URL: https://${DOMAIN}/ VITE_KC_CONFIG: ${KC_CONFIG} + VITE_FILEPOND_TIMEOUT: 10000 # 10 seconds MATOMO_TAG_MANAGER_URL: ${MATOMO_TAG_MANAGER_URL} restart: unless-stopped healthcheck: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 896a0fae..bfbec7fb 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -77,6 +77,7 @@ services: VITE_SECURITY: true VITE_API_URL: https://${DOMAIN}/ VITE_KC_CONFIG: ${KC_CONFIG} + VITE_FILEPOND_TIMEOUT: 10000 # 10 seconds MATOMO_TAG_MANAGER_URL: ${MATOMO_TAG_MANAGER_URL} restart: unless-stopped healthcheck: @@ -130,8 +131,8 @@ services: target: /opt/keycloak/conf/server.key.pem read_only: true - ./custom_keycloak_theme/target/retrocompat-keycloakify-starter-keycloak-theme-5.0.3.jar:/opt/keycloak/providers/retrocompat-keycloakify-starter-keycloak-theme-5.0.3.jar:rw - - ./custom_keycloak_theme/src/main/resources/theme/account-v1:/opt/keycloak/themes/account-v1:rw - - ./custom_keycloak_theme/src/main/resources/theme/keycloakify-starter-variant-1_retrocompat:/opt/keycloak/themes/keycloakify-starter-variant-1_retrocompat:rw + - ./custom_keycloak_theme/src/main/resources/theme/account-v1:/opt/keycloak/themes/account-v1:rw + - ./custom_keycloak_theme/src/main/resources/theme/keycloakify-starter-variant-1_retrocompat:/opt/keycloak/themes/keycloakify-starter-variant-1_retrocompat:rw - ./custom_keycloak_theme/src/main/resources/theme/keycloakify-starter_retrocompat:/opt/keycloak/themes/keycloakify-starter_retrocompat:rw restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index 6448f2bf..52c93382 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,6 +70,7 @@ services: NODE_ENV: production VITE_SECURITY: ${API_SECURITY} VITE_API_URL: http://${DOMAIN}:5000/ + VITE_FILEPOND_TIMEOUT: 7000 # 7 seconds VITE_KC_CONFIG: $KC_CONFIG restart: unless-stopped healthcheck: diff --git a/makefile b/makefile index 8404ee46..4196ff6a 100644 --- a/makefile +++ b/makefile @@ -64,8 +64,8 @@ kc-key: kc-ip: docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' local_keycloak -db-ip: - db_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' convert-pheno-api-db) +db-ip:| + db_ip=$$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' convert-pheno-api-db) && \ echo $${db_ip} mkcert: diff --git a/nginx_mountpoint/replace_env_vars.sh b/nginx_mountpoint/replace_env_vars.sh index 7ddd6abc..da4c5cae 100755 --- a/nginx_mountpoint/replace_env_vars.sh +++ b/nginx_mountpoint/replace_env_vars.sh @@ -13,6 +13,7 @@ echo "=== DEBUG: check if env variables are set ===" echo $VITE_SECURITY echo $VITE_API_URL +echo $VITE_FILEPOND_TIMEOUT echo $VITE_KC_CONFIG echo $MATOMO_TAG_MANAGER_URL echo "=== DEBUG: End of check if env variables are set ===" From 0148aaade15408a4bdecb684dd3f97b9b07282b0 Mon Sep 17 00:00:00 2001 From: Ivo Leist Date: Tue, 12 Dec 2023 00:29:54 +0100 Subject: [PATCH 2/3] removed the on remove handler and replaced it with a useEffect hook --- .../inputFilesPond/InputFilesPond.jsx | 102 ++++++++++-------- convertPheno_server/docker-compose.yml | 27 ++--- convertPheno_server/server/apis/submission.py | 2 + 3 files changed, 77 insertions(+), 54 deletions(-) diff --git a/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx b/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx index 89b996d4..a4509568 100644 --- a/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx +++ b/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx @@ -10,7 +10,7 @@ License: GPL-3.0 license */ -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; import auth from "../../../../../../../../../../Auth"; import { Button, @@ -98,6 +98,28 @@ export default function InputFilesPond(props) { const [files, setFiles] = useState([]); + useEffect(() => { + // to check if one file has been deleted + // if so, delete the file from the uploadedFiles state + const fileNames = files.map((file) => file.filename); + const uploadedFileNames = Object.keys(uploadedFiles); + + const deletedFiles = uploadedFileNames.filter( + (fileName) => !fileNames.includes(fileName) + ); + + if (deletedFiles.length > 0) { + setFilesUploadFinished(false); + setUploadedFiles((prev) => { + const prevCopy = { ...prev }; + deletedFiles.forEach((fileName) => { + delete prevCopy[fileName]; + }); + return prevCopy; + }); + } + }, [files]); + const handleFileUploadFinished = (_, file) => { /* handleFileUploadFinished function @@ -123,15 +145,11 @@ export default function InputFilesPond(props) { return; } - let fileType = "input-file"; - if (filename.includes("dictionary") && fileExtension === "csv") { - fileType = "redcap-dictionary"; - } else if ( - filename.includes("mapping") && - ["yaml", "yml", "json"].includes(fileExtension) - ) { - fileType = "mapping-file"; - } + const fileType = filename.includes("dictionary") && fileExtension === "csv" + ? "redcap-dictionary" + : filename.includes("mapping") && ["yaml", "yml", "json"].includes(fileExtension) + ? "mapping-file" + : "input-file" setUploadedFiles((prev) => { return { @@ -278,38 +296,38 @@ export default function InputFilesPond(props) { }} onprocessfile={handleFileUploadFinished} onprocessfiles={handleAllFilesUploadFinished} - onremovefile={(_, file) => { - setFilesUploadFinished(false); - const fileName = file.filename; - - console.log("files", files); - - // only update state if the file processing was finished (status 2 = IDLE) - // for reference see - // github.com/pqina/filepond-docs/blob/master/content/patterns/API/filepond-object.md#filestatus-enum - - // TODO - // below is not working on the production environment - // It will always retrigger the re-upload of the file - - // Potential solution: - // do not have a setUploadedFile call here - // but rather use the files set by FilePond - // either use this state directly - // or use a useEffect hook to update the uploadedFiles state - // whenever the files state changes - - if ( - file.status === 2 && - file.getMetadata("processingAborted") !== true - ) { - setUploadedFiles((prev) => { - const prevCopy = { ...prev }; - delete prevCopy[fileName]; - return prevCopy; - }); - } - }} + // onremovefile={(_, file) => { + // setFilesUploadFinished(false); + // const fileName = file.filename; + + // console.log("files", files); + + // // only update state if the file processing was finished (status 2 = IDLE) + // // for reference see + // // github.com/pqina/filepond-docs/blob/master/content/patterns/API/filepond-object.md#filestatus-enum + + // // TODO + // // below is not working on the production environment + // // It will always retrigger the re-upload of the file + + // // Potential solution: + // // do not have a setUploadedFile call here + // // but rather use the files set by FilePond + // // either use this state directly + // // or use a useEffect hook to update the uploadedFiles state + // // whenever the files state changes + + // if ( + // file.status === 2 && + // file.getMetadata("processingAborted") !== true + // ) { + // setUploadedFiles((prev) => { + // const prevCopy = { ...prev }; + // delete prevCopy[fileName]; + // return prevCopy; + // }); + // } + // }} // option provided by plugins acceptedFileTypes={acceptedFileTypesMapping[inputFormat]} fileValidateTypeLabelExpectedTypes={ diff --git a/convertPheno_server/docker-compose.yml b/convertPheno_server/docker-compose.yml index 83b16ba9..7a6de9a1 100644 --- a/convertPheno_server/docker-compose.yml +++ b/convertPheno_server/docker-compose.yml @@ -28,19 +28,22 @@ services: # volumes: # - ./data:/data - db: - image: postgres:13-alpine - container_name: convert-pheno-api-db - volumes: - - postgres-data:/var/lib/postgresql/data/ - environment: - - POSTGRES_DB=${API_DB_NAME} - - POSTGRES_USER=${API_DB_USER} - - POSTGRES_PASSWORD=${API_DB_PW} - restart: unless-stopped - healthcheck: - test: pg_isready -U postgres + # db: + # image: postgres:13-alpine + # container_name: convert-pheno-api-db + # volumes: + # - postgres-data:/var/lib/postgresql/data/ + # environment: + # - POSTGRES_DB=${API_DB_NAME} + # - POSTGRES_USER=${API_DB_USER} + # - POSTGRES_PASSWORD=${API_DB_PW} + # restart: unless-stopped + # healthcheck: + # test: pg_isready -U postgres + # run below for local development + # make sure to run it from the root of this folder + # so that the data folder is mounted correctly convert-pheno: user: ${UID} image: manuelrueda/convert-pheno:0.15 diff --git a/convertPheno_server/server/apis/submission.py b/convertPheno_server/server/apis/submission.py index 64b777c7..24bba933 100644 --- a/convertPheno_server/server/apis/submission.py +++ b/convertPheno_server/server/apis/submission.py @@ -396,6 +396,8 @@ def post(self, userid, uuid): with open(out_dir / file_name_mapping["output_name"]) as file: obj = json.load(file) except FileNotFoundError as err: + # The most likely reason for this error is that the + # convert-pheno container volume is mounted incorrectly print(err) errors = {target_format: "File not found"} update_job_status(job, target_format, "failed") From 4c6002675288941dc971d2e19cbd03e5e97626bc Mon Sep 17 00:00:00 2001 From: Ivo Leist Date: Tue, 12 Dec 2023 12:48:02 +0100 Subject: [PATCH 3/3] moved the infering of the file type into its own function --- .../inputFilesPond/InputFilesPond.jsx | 46 +++++++++++++++---- convertPheno_server/docker-compose.yml | 2 +- convertPheno_server/example.env | 2 + convertPheno_server/server/config/config.py | 11 +++++ .../server/utils/convert_pheno.py | 5 +- docker-compose.prod.yml | 1 + docker-compose.yml | 1 + example.env | 2 + 8 files changed, 58 insertions(+), 12 deletions(-) diff --git a/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx b/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx index a4509568..3373319f 100644 --- a/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx +++ b/convertPheno_client/src/code/views/conversion/ components/submission/components/submissionForms/components/fileUpload/components/inputFilesPond/InputFilesPond.jsx @@ -120,6 +120,39 @@ export default function InputFilesPond(props) { } }, [files]); + const inferPotentialFileType = (file) => { + /* + inferPotentialFileType function + + Props: + - filename (string): name of the file that was uploaded + + Functionality: + - tries to infer based on the file name and extension the file type + (input file, dictionary, mapping file) + + Purpose: + - To update the state uploadedFiles with the file that was uploaded + */ + const { filename, fileExtension } = file; + + if ( + filename.includes("dictionary") && + ["csv", "tsv", "txt"].includes(fileExtension) + ) { + return "redcap-dictionary"; + } + + if ( + filename.includes("mapping") && + ["yaml", "yml", "json"].includes(fileExtension) + ) { + return "mapping-file"; + } + + return "input-file"; + }; + const handleFileUploadFinished = (_, file) => { /* handleFileUploadFinished function @@ -131,12 +164,12 @@ export default function InputFilesPond(props) { - tries to infer based on the file name and extension the file type (input file, dictionary, mapping file) - Purpose: - - To update the state uploadedFiles with the file that was uploaded + Purpose: + - To update the state uploadedFiles with the file that was uploaded */ const returnedFileName = JSON.parse(file.serverId).tempFilename; - const { filename, fileExtension } = file; + const { filename } = file; if (filename in uploadedFiles) { toast.error("File already uploaded"); @@ -144,12 +177,7 @@ export default function InputFilesPond(props) { file.abortProcessing(); return; } - - const fileType = filename.includes("dictionary") && fileExtension === "csv" - ? "redcap-dictionary" - : filename.includes("mapping") && ["yaml", "yml", "json"].includes(fileExtension) - ? "mapping-file" - : "input-file" + const fileType = inferPotentialFileType(file); setUploadedFiles((prev) => { return { diff --git a/convertPheno_server/docker-compose.yml b/convertPheno_server/docker-compose.yml index 7a6de9a1..c401cf65 100644 --- a/convertPheno_server/docker-compose.yml +++ b/convertPheno_server/docker-compose.yml @@ -47,7 +47,7 @@ services: convert-pheno: user: ${UID} image: manuelrueda/convert-pheno:0.15 - container_name: convert-pheno + container_name: convert-pheno-container restart: unless-stopped tty: true volumes: diff --git a/convertPheno_server/example.env b/convertPheno_server/example.env index 82a05e64..6fbb49ee 100644 --- a/convertPheno_server/example.env +++ b/convertPheno_server/example.env @@ -14,3 +14,5 @@ API_SECURITY=true # you can get the public Keycloak key in the admin view or by running # bash get_public_kc_key.sh https:///auth/ KC_PUBLIC_KEY=MII.. + +CP_CONTAINER_NAME=convert-pheno-container diff --git a/convertPheno_server/server/config/config.py b/convertPheno_server/server/config/config.py index 707b6122..0e7de4eb 100644 --- a/convertPheno_server/server/config/config.py +++ b/convertPheno_server/server/config/config.py @@ -19,6 +19,15 @@ db_name = environ.get("API_DB_NAME") security = environ.get("API_SECURITY") vscode_debugger = environ.get("DEBUGGER") +cp_container_name = environ.get("CP_CONTAINER_NAME") + +# check if all environment variables are set +if None in [db_user, db_pw, db_host, db_port, db_name, security, cp_container_name]: + raise ValueError( + "Please set the environment variables:" + "API_DB_USER, API_DB_PW, API_DB_HOST, API_DB_PORT, API_DB_NAME" + "API_SECURITY, CP_CONTAINER_NAME" + ) # throw an error if security is not "true" or "false" if security not in ["true", "false"]: @@ -44,3 +53,5 @@ class Config: FLASK_UPLOAD_DIR = Path("../data/uploads/") FLASK_EXAMPLE_DIR = Path("../data/example_in/").resolve() FLASK_OUT_DIR = Path("../data/output/").resolve() + + CP_CONTAINER_NAME = cp_container_name diff --git a/convertPheno_server/server/utils/convert_pheno.py b/convertPheno_server/server/utils/convert_pheno.py index d8f364cd..6549d1a0 100644 --- a/convertPheno_server/server/utils/convert_pheno.py +++ b/convertPheno_server/server/utils/convert_pheno.py @@ -26,10 +26,11 @@ def get_docker_container(): """ # TODO: add error handling client = docker.from_env() + container_name = cfg["CP_CONTAINER_NAME"] try: - container_obj = client.containers.get("convert-pheno") + container_obj = client.containers.get(container_name) except docker.errors.NotFound: - raise ValueError("Docker container convert-pheno not found") + raise ValueError(f"Docker container {container_name} not found") return container_obj diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index bfbec7fb..42f473ce 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -29,6 +29,7 @@ services: UID: ${UID} GID: ${GID} API_SECURITY: true + CP_CONTAINER_NAME: convert-pheno restart: unless-stopped healthcheck: test: curl -f http://0.0.0.0:5000/api/curltest diff --git a/docker-compose.yml b/docker-compose.yml index 52c93382..ba1d1c18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,7 @@ services: UID: ${UID} GID: ${GID} API_SECURITY: ${API_SECURITY} + CP_CONTAINER_NAME: convert-pheno restart: unless-stopped healthcheck: test: curl -f http://0.0.0.0:5000/api/curltest diff --git a/example.env b/example.env index 2b086c99..4bf751b4 100644 --- a/example.env +++ b/example.env @@ -23,3 +23,5 @@ API_DB_NAME=postgres # API configuration ## enable/disable keycloak security API_SECURITY=true + +CONVERT_PHENO_CONTAINER_NAME=convert-pheno