Merge pull request #403 from pepkit/dev

Release 0.14.2

pephub/_version.py

@@ -1 +1 @@
-__version__ = "0.14.1"
+__version__ = "0.14.2"

pephub/const.py

@@ -140,3 +140,9 @@
 )
 
 ARCHIVE_URL_PATH = "https://cloud2.databio.org/pephub/"
+
+MAX_PROCESSED_PROJECT_SIZE = 5000
+
+MAX_STANDARDIZED_PROJECT_SIZE = 100
+
+BEDMS_REPO_URL = "databio/attribute-standardizer-model6"

pephub/routers/api/v1/project.py

@@ -55,6 +55,11 @@
     ConfigResponseModel,
     StandardizerResponse,
 )
+from ....const import (
+    MAX_PROCESSED_PROJECT_SIZE,
+    BEDMS_REPO_URL,
+    MAX_STANDARDIZED_PROJECT_SIZE,
+)
 from .helpers import verify_updated_project
 
 from bedms import AttrStandardizer
@@ -93,7 +98,12 @@ async def get_namespace_projects_list(
     return agent.annotation.get_by_rp_list(registry_paths=paths, admin=namespace_access)
 
 
-@project.get("", summary="Fetch a PEP", response_model=ProjectRawRequest)
+@project.get(
+    "",
+    summary="Fetch a PEP",
+    response_model=ProjectRawRequest,
+    response_model_by_alias=False,
+)
 async def get_a_pep(
     proj: dict = Depends(get_project),
 ):
@@ -110,7 +120,7 @@ async def get_a_pep(
     """
     try:
         raw_project = ProjectRawModel(**proj)
-        return raw_project.model_dump(by_alias=False)
+        return raw_project
     except Exception:
         raise HTTPException(500, "Unexpected project error!")
 
@@ -255,6 +265,11 @@ async def get_pep_samples(
             )
 
         if isinstance(proj, dict):
+            if len(proj["_sample_dict"]) > MAX_PROCESSED_PROJECT_SIZE:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Project is too large. View raw samples, or create a view. Limit is {MAX_PROCESSED_PROJECT_SIZE} samples.",
+                )
             proj = peppy.Project.from_dict(proj)
 
         if format == "json":
@@ -275,6 +290,11 @@ async def get_pep_samples(
             items=df.replace({np.nan: None}).to_dict(orient="records"),
         )
     if isinstance(proj, dict):
+        if len(proj["_sample_dict"]) > MAX_PROCESSED_PROJECT_SIZE:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Project is too large. View raw samples, or create a view. Limit is {MAX_PROCESSED_PROJECT_SIZE} samples.",
+            )
         proj = peppy.Project.from_dict(proj)
     return [sample.to_dict() for sample in proj.samples]
 
@@ -1166,33 +1186,46 @@ def delete_full_history(
     response_model=StandardizerResponse,
 )
 async def get_standardized_cols(
-    pep: peppy.Project = Depends(get_project),
+    pep: dict = Depends(get_project),
     schema: str = "",
 ):
     """
-    Standardize PEP metadata column headers using BEDmess.
+    Standardize PEP metadata column headers using BEDms.
 
-    :param namespace: pep: PEP string to be standardized
+    :param pep: PEP string to be standardized
     :param schema: Schema for AttrStandardizer
 
     :return dict: Standardized results
     """
 
-    if schema == "":
+    if schema == "" or schema not in ["ENCODE", "BEDBASE", "FAIRTRACKS"]:
         raise HTTPException(
-            code=500,
-            detail="Schema is required! Available schemas are ENCODE and Fairtracks",
+            status_code=404,
+            detail="Schema not available! Available schemas are ENCODE, BEDBASE and FAIRTRACKS.",
+        )
+
+    if len(pep["_sample_dict"]) > MAX_STANDARDIZED_PROJECT_SIZE:
+        # raise HTTPException(
+        #     status_code=400,
+        #     detail=f"Project is too large. Cannot standardize. "
+        #            f"Limit is {MAX_STANDARDIZED_PROJECT_SIZE} samples.",
+        # )
+        prj = peppy.Project.from_dict(
+            {
+                "_config": pep["_config"],
+                "_sample_dict": pep["_sample_dict"][:50],
+            }
         )
-        return {}
-
-    prj = peppy.Project.from_dict(pep)
-    model = AttrStandardizer(schema)
+    else:
+        prj = peppy.Project.from_dict(pep)
+    model = AttrStandardizer(repo_id=BEDMS_REPO_URL, model_name=schema.lower())
 
     try:
         results = model.standardize(pep=prj)
-    except Exception:
+    except Exception as e:
+        _LOGGER.error(f"Error standardizing PEP. {e}")
         raise HTTPException(
-            code=400,
+            status_code=400,
             detail=f"Error standardizing PEP.",
         )
 

pephub/routers/eido/eido.py

@@ -16,6 +16,7 @@
 
 from ...dependencies import DEFAULT_TAG, get_db
 from ...helpers import parse_user_file_upload, split_upload_files_on_init_file
+from ...const import MAX_PROCESSED_PROJECT_SIZE
 
 schemas_url = "https://schema.databio.org/list.json"
 schemas_to_test = requests.get(schemas_url).json()
@@ -84,6 +85,16 @@ async def validate(
     if pep_registry is not None:
         namespace, name, tag = registry_path_converter(pep_registry)
         tag = tag or DEFAULT_TAG
+
+        pep_annot = agent.annotation.get(namespace=namespace, name=name, tag=tag)
+
+        if pep_annot.results[0].number_of_samples > MAX_PROCESSED_PROJECT_SIZE:
+            return {
+                "valid": False,
+                "error_type": "Project size",
+                "errors": ["Project is too large. Can't validate."],
+            }
+
         p = agent.project.get(namespace, name, tag, raw=False)
     else:
         init_file = parse_user_file_upload(pep_files)

requirements/requirements-all.txt

@@ -16,4 +16,4 @@ fastembed
 numpy<2.0.0
 slowapi
 cachetools>=4.2.4
-bedms>=0.1.0
+bedms>=0.2.0

web/src/components/forms/components/pep-search-dropdown.tsx

@@ -13,7 +13,7 @@ interface Props {
 }
 
 export const PepSearchDropdown = (props: Props) => {
-  const { value, onChange, namespace } = props;
+  const { value, onChange, namespace, type } = props;
 
   // value always has the format: namespace/project:tag
   // so we need to split it to get the namespace, project, and tag
@@ -72,7 +72,7 @@ export const PepSearchDropdown = (props: Props) => {
           borderColor: '#dee2e6'
         })
       }}
-      placeholder="Search for PEPs"
+      placeholder={type === 'pop' ? "Search for POPs" : "Search for PEPs"}
       menuPlacement="bottom"
       controlShouldRenderValue={true}
     />

web/src/components/modals/add-to-pop.tsx

@@ -9,6 +9,11 @@ import { useSampleTable } from '../../hooks/queries/useSampleTable';
 import { extractErrorMessage } from '../../utils/etc';
 import { PepSearchDropdown } from '../forms/components/pep-search-dropdown';
 import { LoadingSpinner } from '../spinners/loading-spinner';
+import { useTotalProjectChangeMutation } from '../../hooks/mutations/useTotalProjectChangeMutation';
+import { useProjectConfig } from '../../hooks/queries/useProjectConfig';
+import { useSubsampleTable } from '../../hooks/queries/useSubsampleTable';
+
+
 
 interface Props {
   show: boolean;
@@ -32,13 +37,17 @@ export const AddToPOPModal: FC<Props> = (props) => {
   // derived from project
   const [projectName, tag] = project?.split('/')[1].split(':') || [undefined, undefined];
 
-  // I run data validation in the actual button click, so im not doing it here
-  const { data: currentSampleTable } = useSampleTable({
+  const { data: popConfig } = useProjectConfig(namespace, projectName!, tag);
+  const { data: popSampleTable } = useSampleTable({
     namespace,
     project: projectName!,
     tag: tag,
+    enabled: true,
   });
-  const { isPending: isSampleTablePending, submit } = useSampleTableMutation(namespace, projectName!, tag!);
+  const { data: popSubSampleTable } = useSubsampleTable(namespace, projectName!, tag);
+
+  // const { isPending: isSampleTablePending, submit } = useSampleTableMutation(namespace, projectName!, tag!);
+  const { isPending: isSampleTablePending, submit } = useTotalProjectChangeMutation(namespace, projectName!, tag!);
 
   const onCancel = () => {
     setNamespace(user!.login);
@@ -47,47 +56,43 @@ export const AddToPOPModal: FC<Props> = (props) => {
   };
 
   const onAdd = () => {
-    if (!projectName || !tag) {
+    if (!projectName) {
       toast.error('Please select a project to add to the POP.');
       return;
     }
-    if (!currentSampleTable) {
+    if (!popSampleTable) {
       toast.error('There was an issue fetching the current sample table for the selected POP.');
       return;
     }
     if (
-      currentSampleTable.items.includes({
-        namespace: namespaceToAdd,
-        project: projectToAdd,
-        tag: tagToAdd,
-      })
+      popSampleTable?.items?.some(item => 
+        item.sample_name === `${namespaceToAdd}/${projectToAdd}:${tagToAdd}`
+      )
     ) {
       toast.error('This project is already in the POP!');
       return;
     }
-
-    // finally add the project to the pop if it passes all the checks
-    submit(
-      [
-        ...currentSampleTable.items,
-        {
-          sample_name: `${namespaceToAdd}/${projectToAdd}:${tagToAdd}`,
-          namespace: namespaceToAdd,
-          name: projectToAdd,
-          tag: tagToAdd,
-        },
-      ],
-      {
-        onSuccess: () => {
-          toast.success('Successfully added project to POP!');
-          onCancel();
-        },
-        onError: (err: AxiosError) => {
-          const errorMessage = extractErrorMessage(err);
-          toast.error(`There was an issue adding the project to the POP" ${errorMessage}`);
-        },
-      },
-    );
+    let newPeps = popSampleTable?.items || [];
+    newPeps?.push({
+      sample_name: `${namespaceToAdd}/${projectToAdd}:${tagToAdd}`,
+      namespace: namespaceToAdd,
+      name: projectToAdd,
+      tag: tagToAdd,
+    });
+    submit({
+      config: popConfig?.config,
+      samples: newPeps,
+      subsamples: popSubSampleTable?.items,
+    }, {
+      // onSuccess: () => {
+      //   toast.success('Successfully added project to POP!');
+      //   onCancel();
+      // },
+      // onError: (err: AxiosError) => {
+      //   const errorMessage = extractErrorMessage(err);
+      //   toast.error(`There was an issue adding the project to the POP: ${errorMessage}`);
+      // },
+    });
   };
 
   useEffect(() => {
@@ -129,7 +134,7 @@ export const AddToPOPModal: FC<Props> = (props) => {
           <button
             onClick={onAdd}
             className="btn btn-success"
-            disabled={!projectName || !tag || !currentSampleTable || isSampleTablePending}
+            // disabled={!projectName || !tag || !currentSampleTable || isSampleTablePending}
           >
             {isSampleTablePending ? (
               <Fragment>

web/src/components/modals/validation-result.tsx

@@ -70,29 +70,27 @@ export const ValidationResultModal = (props: Props) => {
               )}
             </>
           ) : (
-            <span className="d-flex align-items-center gap-1">
-              Select a Schema
-            </span>
+            <span className="d-flex align-items-center gap-1">Select a Schema</span>
           )}
         </h1>
       </Modal.Header>
       <Modal.Body>
         {currentSchema && (
           <>
-          {validationResult?.valid ? (
-            <p>Your PEP is valid against the schema.</p>
-          ) : (
-            <Fragment>
-              <p>You PEP is invalid against the schema.</p>
-              <p>Validation result:</p>
-              <pre>
-                <code>{JSON.stringify(validationResult, null, 2)}</code>
-              </pre>
-            </Fragment>
-          )}
+            {validationResult?.valid ? (
+              <p>Your PEP is valid against the schema.</p>
+            ) : (
+              <Fragment>
+                <p>Your PEP is invalid against the schema.</p>
+                <p>Validation result:</p>
+                <pre>
+                  <code>{JSON.stringify(validationResult, null, 2)}</code>
+                </pre>
+              </Fragment>
+            )}
           </>
         )}
-          
+
         <form className="mb-1">
           {currentSchema ? (
             <label className="mt-1 fw-bold">Change schemas here:</label>
@@ -126,7 +124,7 @@ export const ValidationResultModal = (props: Props) => {
       </Modal.Body>
       <Modal.Footer>
         <div className="d-flex align-items-center justify-content-between w-100">
-          { currentSchema && (
+          {currentSchema && (
             <div className="d-flex align-items-center">
               <a href={`/schemas/${props.currentSchema}`}>
                 <button className="btn btn-sm btn-outline-dark">

web/src/components/project/project-validation-and-edit-buttons.tsx

@@ -18,13 +18,17 @@ type ProjectValidationAndEditButtonsProps = {
   filteredSamples: string[];
 };
 
+const MAX_SAMPLES_FOR_VALIDATION = 5000;
+
 export const ProjectValidationAndEditButtons = (props: ProjectValidationAndEditButtonsProps) => {
   const { isDirty, isUpdatingProject, reset, handleSubmit, filteredSamples } = props;
   const { user } = useSession();
 
   const { namespace, projectName, tag } = useProjectPage();
 
   const { data: projectInfo } = useProjectAnnotation(namespace, projectName, tag);
+  const shouldValidate: boolean = (projectInfo?.number_of_samples || 0) > MAX_SAMPLES_FOR_VALIDATION;
+
   const projectValidationQuery = useValidation({
     pepRegistry: `${namespace}/${projectName}:${tag}`,
     schema_registry: projectInfo?.pep_schema || 'pep/2.0.0',
@@ -48,6 +52,7 @@ export const ProjectValidationAndEditButtons = (props: ProjectValidationAndEditB
                 schemaRegistry={projectSchema}
                 isValidating={projectValidationQuery.isLoading}
                 validationResult={validationResult}
+                shouldValidate={shouldValidate}
               />
             </div>
             <div className="ps-1">