From fad42481feef68ca16da823bee00a26d3f6b6a66 Mon Sep 17 00:00:00 2001 From: Sam Park Date: Mon, 21 Oct 2024 11:06:05 -0400 Subject: [PATCH 1/5] disable sample table sorting for #400 --- web/src/components/tables/sample-table.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/src/components/tables/sample-table.tsx b/web/src/components/tables/sample-table.tsx index d486141c..d7be0ee8 100644 --- a/web/src/components/tables/sample-table.tsx +++ b/web/src/components/tables/sample-table.tsx @@ -149,7 +149,8 @@ export const SampleTable = (props: Props) => { 'copy', 'cut', ]} - multiColumnSorting={true} + multiColumnSorting={false} + columnSorting={false} filters={true} rowHeaders={true} beforeRenderer={addClassesToRows} From 26b5237e9f92f4af579cf6d09bc016be45d5cdd1 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 21 Nov 2024 11:53:41 -0500 Subject: [PATCH 2/5] Fixed #401 --- pephub/const.py | 2 + pephub/routers/api/v1/project.py | 20 ++++- pephub/routers/eido/eido.py | 11 +++ .../components/modals/validation-result.tsx | 30 ++++--- .../project-validation-and-edit-buttons.tsx | 5 ++ .../project/validation/validation-result.tsx | 82 +++++++++---------- 6 files changed, 91 insertions(+), 59 deletions(-) diff --git a/pephub/const.py b/pephub/const.py index dcc91bdd..18c1ad76 100644 --- a/pephub/const.py +++ b/pephub/const.py @@ -140,3 +140,5 @@ ) ARCHIVE_URL_PATH = "https://cloud2.databio.org/pephub/" + +MAX_PROCESSED_PROJECT_SIZE = 5000 diff --git a/pephub/routers/api/v1/project.py b/pephub/routers/api/v1/project.py index 5c845600..5e7a44d7 100644 --- a/pephub/routers/api/v1/project.py +++ b/pephub/routers/api/v1/project.py @@ -55,6 +55,7 @@ ConfigResponseModel, StandardizerResponse, ) +from ....const import MAX_PROCESSED_PROJECT_SIZE from .helpers import verify_updated_project from bedms import AttrStandardizer @@ -93,7 +94,12 @@ async def get_namespace_projects_list( return agent.annotation.get_by_rp_list(registry_paths=paths, admin=namespace_access) -@project.get("", summary="Fetch a PEP", response_model=ProjectRawRequest) +@project.get( + "", + summary="Fetch a PEP", + response_model=ProjectRawRequest, + response_model_by_alias=False, +) async def get_a_pep( proj: dict = Depends(get_project), ): @@ -110,7 +116,7 @@ async def get_a_pep( """ try: raw_project = ProjectRawModel(**proj) - return raw_project.model_dump(by_alias=False) + return raw_project except Exception: raise HTTPException(500, "Unexpected project error!") @@ -255,6 +261,11 @@ async def get_pep_samples( ) if isinstance(proj, dict): + if len(proj["_sample_dict"]) > MAX_PROCESSED_PROJECT_SIZE: + raise HTTPException( + status_code=400, + detail=f"Project is too large. View raw samples, or create a view. Limit is {MAX_PROCESSED_PROJECT_SIZE} samples.", + ) proj = peppy.Project.from_dict(proj) if format == "json": @@ -275,6 +286,11 @@ async def get_pep_samples( items=df.replace({np.nan: None}).to_dict(orient="records"), ) if isinstance(proj, dict): + if len(proj["_sample_dict"]) > MAX_PROCESSED_PROJECT_SIZE: + raise HTTPException( + status_code=400, + detail=f"Project is too large. View raw samples, or create a view. Limit is {MAX_PROCESSED_PROJECT_SIZE} samples.", + ) proj = peppy.Project.from_dict(proj) return [sample.to_dict() for sample in proj.samples] diff --git a/pephub/routers/eido/eido.py b/pephub/routers/eido/eido.py index c26f40e9..e313c1b4 100644 --- a/pephub/routers/eido/eido.py +++ b/pephub/routers/eido/eido.py @@ -16,6 +16,7 @@ from ...dependencies import DEFAULT_TAG, get_db from ...helpers import parse_user_file_upload, split_upload_files_on_init_file +from ...const import MAX_PROCESSED_PROJECT_SIZE schemas_url = "https://schema.databio.org/list.json" schemas_to_test = requests.get(schemas_url).json() @@ -84,6 +85,16 @@ async def validate( if pep_registry is not None: namespace, name, tag = registry_path_converter(pep_registry) tag = tag or DEFAULT_TAG + + pep_annot = agent.annotation.get(namespace=namespace, name=name, tag=tag) + + if pep_annot.results[0].number_of_samples > MAX_PROCESSED_PROJECT_SIZE: + return { + "valid": False, + "error_type": "Project size", + "errors": ["Project is too large. Can't validate."], + } + p = agent.project.get(namespace, name, tag, raw=False) else: init_file = parse_user_file_upload(pep_files) diff --git a/web/src/components/modals/validation-result.tsx b/web/src/components/modals/validation-result.tsx index f542d5a9..61a70d7f 100644 --- a/web/src/components/modals/validation-result.tsx +++ b/web/src/components/modals/validation-result.tsx @@ -70,29 +70,27 @@ export const ValidationResultModal = (props: Props) => { )} ) : ( - - Select a Schema - + Select a Schema )} {currentSchema && ( <> - {validationResult?.valid ? ( -

Your PEP is valid against the schema.

- ) : ( - -

You PEP is invalid against the schema.

-

Validation result:

- 
-                {JSON.stringify(validationResult, null, 2)}
-
- - )} + {validationResult?.valid ? ( +

Your PEP is valid against the schema.

+ ) : ( + +

Your PEP is invalid against the schema.

+

Validation result:

+ 
+                  {JSON.stringify(validationResult, null, 2)}
+
+ + )} )} - +
{currentSchema ? ( @@ -126,7 +124,7 @@ export const ValidationResultModal = (props: Props) => {
- { currentSchema && ( + {currentSchema && (
diff --git a/web/src/components/project/validation/validation-result.tsx b/web/src/components/project/validation/validation-result.tsx index 43dfef6a..f23dfca5 100644 --- a/web/src/components/project/validation/validation-result.tsx +++ b/web/src/components/project/validation/validation-result.tsx @@ -2,23 +2,25 @@ import { useState } from 'react'; import { OverlayTrigger, Tooltip } from 'react-bootstrap'; import { useValidation } from '../../../hooks/queries/useValidation'; -import { ValidationResultModal } from '../../modals/validation-result'; import { StatusIcon } from '../../badges/status-icons'; - +import { ValidationResultModal } from '../../modals/validation-result'; type Props = { schemaRegistry: string | undefined; isValidating: boolean; validationResult: ReturnType['data']; + shouldValidate: boolean; }; export const ValidationResult = (props: Props) => { - const { validationResult, isValidating, schemaRegistry } = props; + const { validationResult, isValidating, schemaRegistry, shouldValidate } = props; const [validationModalIsOpen, setValidationModalIsOpen] = useState(false); let wrapperClassName = 'py-1 px-2 rounded-1 bg-opacity-10 validation-button'; - if (isValidating) { + if (!shouldValidate) { + wrapperClassName += ' border border-warning text-warning bg-warning'; + } else if (isValidating) { wrapperClassName += ' border border-warning text-warning bg-warning'; } else if (validationResult?.valid) { wrapperClassName += ' border border-success text-success bg-success'; @@ -37,45 +39,43 @@ export const ValidationResult = (props: Props) => { delay={{ show: 250, hide: 500 }} trigger={['hover']} > - - { schemaRegistry ? ( - - ) : ( - - )} - + className={wrapperClassName} + > +
+ {isValidating || !shouldValidate ? ( + + ) : validationResult?.valid ? ( + + ) : ( + + )} + + {schemaRegistry} + +
+ + ) : ( + + )} Date: Thu, 21 Nov 2024 17:59:02 -0500 Subject: [PATCH 3/5] validation button color change? --- web/src/components/project/validation/validation-result.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/components/project/validation/validation-result.tsx b/web/src/components/project/validation/validation-result.tsx index f23dfca5..354f344a 100644 --- a/web/src/components/project/validation/validation-result.tsx +++ b/web/src/components/project/validation/validation-result.tsx @@ -18,7 +18,7 @@ export const ValidationResult = (props: Props) => { const [validationModalIsOpen, setValidationModalIsOpen] = useState(false); let wrapperClassName = 'py-1 px-2 rounded-1 bg-opacity-10 validation-button'; - if (!shouldValidate) { + if (shouldValidate) { wrapperClassName += ' border border-warning text-warning bg-warning'; } else if (isValidating) { wrapperClassName += ' border border-warning text-warning bg-warning'; @@ -48,7 +48,7 @@ export const ValidationResult = (props: Props) => { className={wrapperClassName} >
- {isValidating || !shouldValidate ? ( + {isValidating || shouldValidate ? ( ) : validationResult?.valid ? ( From abe3de09045c6dd96febe738bc965175036f460d Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 3 Dec 2024 15:57:03 -0500 Subject: [PATCH 4/5] Updated bedms to new version + stability improvements --- pephub/_version.py | 2 +- pephub/const.py | 4 +++ pephub/routers/api/v1/project.py | 43 +++++++++++++++++++++---------- requirements/requirements-all.txt | 2 +- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/pephub/_version.py b/pephub/_version.py index f075dd36..745162e7 100644 --- a/pephub/_version.py +++ b/pephub/_version.py @@ -1 +1 @@ -__version__ = "0.14.1" +__version__ = "0.14.2" diff --git a/pephub/const.py b/pephub/const.py index 18c1ad76..89af9111 100644 --- a/pephub/const.py +++ b/pephub/const.py @@ -142,3 +142,7 @@ ARCHIVE_URL_PATH = "https://cloud2.databio.org/pephub/" MAX_PROCESSED_PROJECT_SIZE = 5000 + +MAX_STANDARDIZED_PROJECT_SIZE = 100 + +BEDMS_REPO_URL = "databio/attribute-standardizer-model6" diff --git a/pephub/routers/api/v1/project.py b/pephub/routers/api/v1/project.py index 5e7a44d7..5d873229 100644 --- a/pephub/routers/api/v1/project.py +++ b/pephub/routers/api/v1/project.py @@ -55,7 +55,11 @@ ConfigResponseModel, StandardizerResponse, ) -from ....const import MAX_PROCESSED_PROJECT_SIZE +from ....const import ( + MAX_PROCESSED_PROJECT_SIZE, + BEDMS_REPO_URL, + MAX_STANDARDIZED_PROJECT_SIZE, +) from .helpers import verify_updated_project from bedms import AttrStandardizer @@ -1182,33 +1186,46 @@ def delete_full_history( response_model=StandardizerResponse, ) async def get_standardized_cols( - pep: peppy.Project = Depends(get_project), + pep: dict = Depends(get_project), schema: str = "", ): """ - Standardize PEP metadata column headers using BEDmess. + Standardize PEP metadata column headers using BEDms. - :param namespace: pep: PEP string to be standardized + :param pep: PEP string to be standardized :param schema: Schema for AttrStandardizer :return dict: Standardized results """ - if schema == "": + if schema == "" or schema not in ["ENCODE", "BEDBASE", "FAIRTRACKS"]: raise HTTPException( - code=500, - detail="Schema is required! Available schemas are ENCODE and Fairtracks", + status_code=404, + detail="Schema not available! Available schemas are ENCODE, BEDBASE and FAIRTRACKS.", + ) + + if len(pep["_sample_dict"]) > MAX_STANDARDIZED_PROJECT_SIZE: + # raise HTTPException( + # status_code=400, + # detail=f"Project is too large. Cannot standardize. " + # f"Limit is {MAX_STANDARDIZED_PROJECT_SIZE} samples.", + # ) + prj = peppy.Project.from_dict( + { + "_config": pep["_config"], + "_sample_dict": pep["_sample_dict"][:50], + } ) - return {} - - prj = peppy.Project.from_dict(pep) - model = AttrStandardizer(schema) + else: + prj = peppy.Project.from_dict(pep) + model = AttrStandardizer(repo_id=BEDMS_REPO_URL, model_name=schema.lower()) try: results = model.standardize(pep=prj) - except Exception: + except Exception as e: + _LOGGER.error(f"Error standardizing PEP. {e}") raise HTTPException( - code=400, + status_code=400, detail=f"Error standardizing PEP.", ) diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index d7903965..25631478 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -16,4 +16,4 @@ fastembed numpy<2.0.0 slowapi cachetools>=4.2.4 -bedms>=0.1.0 \ No newline at end of file +bedms>=0.2.0 \ No newline at end of file From 780f77cef23bbb122329b09f29f7826ba09fa725 Mon Sep 17 00:00:00 2001 From: Sam Park Date: Tue, 3 Dec 2024 17:19:45 -0500 Subject: [PATCH 5/5] fix add to pop from pep dropdown --- .../forms/components/pep-search-dropdown.tsx | 4 +- web/src/components/modals/add-to-pop.tsx | 73 ++++++++++--------- web/src/hooks/queries/useProjectConfig.ts | 2 +- web/src/hooks/queries/useSubsampleTable.ts | 2 +- 4 files changed, 43 insertions(+), 38 deletions(-) diff --git a/web/src/components/forms/components/pep-search-dropdown.tsx b/web/src/components/forms/components/pep-search-dropdown.tsx index 9492c58b..bd4ff0ae 100644 --- a/web/src/components/forms/components/pep-search-dropdown.tsx +++ b/web/src/components/forms/components/pep-search-dropdown.tsx @@ -13,7 +13,7 @@ interface Props { } export const PepSearchDropdown = (props: Props) => { - const { value, onChange, namespace } = props; + const { value, onChange, namespace, type } = props; // value always has the format: namespace/project:tag // so we need to split it to get the namespace, project, and tag @@ -72,7 +72,7 @@ export const PepSearchDropdown = (props: Props) => { borderColor: '#dee2e6' }) }} - placeholder="Search for PEPs" + placeholder={type === 'pop' ? "Search for POPs" : "Search for PEPs"} menuPlacement="bottom" controlShouldRenderValue={true} /> diff --git a/web/src/components/modals/add-to-pop.tsx b/web/src/components/modals/add-to-pop.tsx index 44b66d38..dcd37057 100644 --- a/web/src/components/modals/add-to-pop.tsx +++ b/web/src/components/modals/add-to-pop.tsx @@ -9,6 +9,11 @@ import { useSampleTable } from '../../hooks/queries/useSampleTable'; import { extractErrorMessage } from '../../utils/etc'; import { PepSearchDropdown } from '../forms/components/pep-search-dropdown'; import { LoadingSpinner } from '../spinners/loading-spinner'; +import { useTotalProjectChangeMutation } from '../../hooks/mutations/useTotalProjectChangeMutation'; +import { useProjectConfig } from '../../hooks/queries/useProjectConfig'; +import { useSubsampleTable } from '../../hooks/queries/useSubsampleTable'; + + interface Props { show: boolean; @@ -32,13 +37,17 @@ export const AddToPOPModal: FC = (props) => { // derived from project const [projectName, tag] = project?.split('/')[1].split(':') || [undefined, undefined]; - // I run data validation in the actual button click, so im not doing it here - const { data: currentSampleTable } = useSampleTable({ + const { data: popConfig } = useProjectConfig(namespace, projectName!, tag); + const { data: popSampleTable } = useSampleTable({ namespace, project: projectName!, tag: tag, + enabled: true, }); - const { isPending: isSampleTablePending, submit } = useSampleTableMutation(namespace, projectName!, tag!); + const { data: popSubSampleTable } = useSubsampleTable(namespace, projectName!, tag); + + // const { isPending: isSampleTablePending, submit } = useSampleTableMutation(namespace, projectName!, tag!); + const { isPending: isSampleTablePending, submit } = useTotalProjectChangeMutation(namespace, projectName!, tag!); const onCancel = () => { setNamespace(user!.login); @@ -47,47 +56,43 @@ export const AddToPOPModal: FC = (props) => { }; const onAdd = () => { - if (!projectName || !tag) { + if (!projectName) { toast.error('Please select a project to add to the POP.'); return; } - if (!currentSampleTable) { + if (!popSampleTable) { toast.error('There was an issue fetching the current sample table for the selected POP.'); return; } if ( - currentSampleTable.items.includes({ - namespace: namespaceToAdd, - project: projectToAdd, - tag: tagToAdd, - }) + popSampleTable?.items?.some(item => + item.sample_name === `${namespaceToAdd}/${projectToAdd}:${tagToAdd}` + ) ) { toast.error('This project is already in the POP!'); return; } - - // finally add the project to the pop if it passes all the checks - submit( - [ - ...currentSampleTable.items, - { - sample_name: `${namespaceToAdd}/${projectToAdd}:${tagToAdd}`, - namespace: namespaceToAdd, - name: projectToAdd, - tag: tagToAdd, - }, - ], - { - onSuccess: () => { - toast.success('Successfully added project to POP!'); - onCancel(); - }, - onError: (err: AxiosError) => { - const errorMessage = extractErrorMessage(err); - toast.error(`There was an issue adding the project to the POP" ${errorMessage}`); - }, - }, - ); + let newPeps = popSampleTable?.items || []; + newPeps?.push({ + sample_name: `${namespaceToAdd}/${projectToAdd}:${tagToAdd}`, + namespace: namespaceToAdd, + name: projectToAdd, + tag: tagToAdd, + }); + submit({ + config: popConfig?.config, + samples: newPeps, + subsamples: popSubSampleTable?.items, + }, { + // onSuccess: () => { + // toast.success('Successfully added project to POP!'); + // onCancel(); + // }, + // onError: (err: AxiosError) => { + // const errorMessage = extractErrorMessage(err); + // toast.error(`There was an issue adding the project to the POP: ${errorMessage}`); + // }, + }); }; useEffect(() => { @@ -129,7 +134,7 @@ export const AddToPOPModal: FC = (props) => {