Merge pull request #317 from arayabrain/feature/microscope-node

implement microscope data reader nodes
oist · Mar 26, 2024 · 752a944 · 752a944
2 parents dd2d8f2 + a623541
commit 752a944
Show file tree

Hide file tree

Showing 28 changed files with 265 additions and 9 deletions.
diff --git a/frontend/src/api/files/Files.ts b/frontend/src/api/files/Files.ts
@@ -10,6 +10,7 @@ export const FILE_TREE_TYPE_SET = {
   FLUO: "fluo",
   BEHAVIOR: "behavior",
   MATLAB: "matlab",
+  MICROSCOPE: "microscope",
   ALL: "all",
 } as const
 

diff --git a/frontend/src/components/Workspace/FlowChart/FlowChartNode/FileSelect.tsx b/frontend/src/components/Workspace/FlowChart/FlowChartNode/FileSelect.tsx
@@ -305,6 +305,8 @@ function getFileInputAccept(fileType: FILE_TREE_TYPE | undefined) {
       return ".csv"
     case FILE_TREE_TYPE_SET.HDF5:
       return ".hdf5,.nwb"
+    case FILE_TREE_TYPE_SET.MICROSCOPE:
+      return ".nd2,.oir,.isxd,.thor.zip"
     default:
       return undefined
   }

diff --git a/frontend/src/components/Workspace/FlowChart/FlowChartNode/MicroscopeFileNode.tsx b/frontend/src/components/Workspace/FlowChart/FlowChartNode/MicroscopeFileNode.tsx
@@ -0,0 +1,82 @@
+import { memo } from "react"
+import { useSelector, useDispatch } from "react-redux"
+import { Handle, Position, NodeProps } from "reactflow"
+
+import { FileSelect } from "components/Workspace/FlowChart/FlowChartNode/FileSelect"
+import {
+  toHandleId,
+  isValidConnection,
+} from "components/Workspace/FlowChart/FlowChartNode/FlowChartUtils"
+import { useHandleColor } from "components/Workspace/FlowChart/FlowChartNode/HandleColorHook"
+import { NodeContainer } from "components/Workspace/FlowChart/FlowChartNode/NodeContainer"
+import { HANDLE_STYLE } from "const/flowchart"
+import { deleteFlowNodeById } from "store/slice/FlowElement/FlowElementSlice"
+import { setInputNodeFilePath } from "store/slice/InputNode/InputNodeActions"
+import {
+  selectMicroscopeInputNodeSelectedFilePath,
+  selectInputNodeDefined,
+} from "store/slice/InputNode/InputNodeSelectors"
+import { FILE_TYPE_SET } from "store/slice/InputNode/InputNodeType"
+
+export const MicroscopeFileNode = memo(function MicroscopeFileNode(
+  element: NodeProps,
+) {
+  const defined = useSelector(selectInputNodeDefined(element.id))
+  if (defined) {
+    return <MicroscopeFileNodeImple {...element} />
+  } else {
+    return null
+  }
+})
+
+const MicroscopeFileNodeImple = memo(function MicroscopeFileNodeImple({
+  id: nodeId,
+  selected: elementSelected,
+}: NodeProps) {
+  const dispatch = useDispatch()
+  const filePath = useSelector(
+    selectMicroscopeInputNodeSelectedFilePath(nodeId),
+  )
+  const onChangeFilePath = (path: string) => {
+    dispatch(setInputNodeFilePath({ nodeId, filePath: path }))
+  }
+
+  const returnType = "MicroscopeData"
+  const microscopeColor = useHandleColor(returnType)
+
+  const onClickDeleteIcon = () => {
+    dispatch(deleteFlowNodeById(nodeId))
+  }
+
+  return (
+    <NodeContainer nodeId={nodeId} selected={elementSelected}>
+      <button
+        className="flowbutton"
+        onClick={onClickDeleteIcon}
+        style={{ color: "black", position: "absolute", top: -10, right: 10 }}
+      >
+        ×
+      </button>
+      <FileSelect
+        nodeId={nodeId}
+        onChangeFilePath={(path) => {
+          if (!Array.isArray(path)) {
+            onChangeFilePath(path)
+          }
+        }}
+        fileType={FILE_TYPE_SET.MICROSCOPE}
+        filePath={filePath ?? ""}
+      />
+      <Handle
+        type="source"
+        position={Position.Right}
+        id={toHandleId(nodeId, "microscope", returnType)}
+        style={{
+          ...HANDLE_STYLE,
+          background: microscopeColor,
+        }}
+        isValidConnection={isValidConnection}
+      />
+    </NodeContainer>
+  )
+})
diff --git a/frontend/src/components/Workspace/FlowChart/FlowChartNode/ReactFlowNodeTypesConst.ts b/frontend/src/components/Workspace/FlowChart/FlowChartNode/ReactFlowNodeTypesConst.ts
@@ -6,6 +6,7 @@ import { FluoFileNode } from "components/Workspace/FlowChart/FlowChartNode/FluoF
 import { HDF5FileNode } from "components/Workspace/FlowChart/FlowChartNode/HDF5FileNode"
 import { ImageFileNode } from "components/Workspace/FlowChart/FlowChartNode/ImageFileNode"
 import { MatlabFileNode } from "components/Workspace/FlowChart/FlowChartNode/MatlabFileNode"
+import { MicroscopeFileNode } from "components/Workspace/FlowChart/FlowChartNode/MicroscopeFileNode"
 
 export const reactFlowNodeTypes = {
   ImageFileNode,
@@ -15,6 +16,7 @@ export const reactFlowNodeTypes = {
   AlgorithmNode,
   FluoFileNode,
   BehaviorFileNode,
+  MicroscopeFileNode,
 } as const
 
 export const reactFlowEdgeTypes = {

diff --git a/frontend/src/components/Workspace/FlowChart/TreeView.tsx b/frontend/src/components/Workspace/FlowChart/TreeView.tsx
@@ -119,6 +119,11 @@ export const AlgorithmTreeView = memo(function AlgorithmTreeView() {
           nodeName={"matlabData"}
           fileType={FILE_TYPE_SET.MATLAB}
         />
+        <InputNodeComponent
+          fileName={"microscope"}
+          nodeName={"microscopeData"}
+          fileType={FILE_TYPE_SET.MICROSCOPE}
+        />
       </TreeItem>
       <TreeItem nodeId="Algorithm" label="Algorithm">
         {Object.entries(algoList).map(([name, node], i) => (
@@ -179,6 +184,10 @@ const InputNodeComponent = memo(function InputNodeComponent({
           reactFlowNodeType = REACT_FLOW_NODE_TYPE_KEY.MatlabFileNode
           fileType = FILE_TYPE_SET.MATLAB
           break
+        case FILE_TYPE_SET.MICROSCOPE:
+          reactFlowNodeType = REACT_FLOW_NODE_TYPE_KEY.MicroscopeFileNode
+          fileType = FILE_TYPE_SET.MICROSCOPE
+          break
       }
       const newNode = {
         id: `input_${getNanoId()}`,

diff --git a/frontend/src/components/Workspace/Visualize/FilePathSelect.tsx b/frontend/src/components/Workspace/Visualize/FilePathSelect.tsx
@@ -228,6 +228,7 @@ function toDataTypeFromFileType(fileType: FILE_TYPE) {
     case FILE_TYPE_SET.BEHAVIOR:
       return DATA_TYPE_SET.BEHAVIOR
     case FILE_TYPE_SET.MATLAB:
+    case FILE_TYPE_SET.MICROSCOPE:
       return DATA_TYPE_SET.MATLAB
   }
 }
diff --git a/frontend/src/const/flowchart.ts b/frontend/src/const/flowchart.ts
@@ -34,6 +34,7 @@ export const REACT_FLOW_NODE_TYPE_KEY = {
   AlgorithmNode: "AlgorithmNode",
   BehaviorFileNode: "BehaviorFileNode",
   MatlabFileNode: "MatlabFileNode",
+  MicroscopeFileNode: "MicroscopeFileNode",
 } as const
 
 export type REACT_FLOW_NODE_TYPE =

diff --git a/frontend/src/store/slice/HandleTypeColor/HandleTypeColorSlice.ts b/frontend/src/store/slice/HandleTypeColor/HandleTypeColorSlice.ts
@@ -18,6 +18,7 @@ const initialState: HandleTypeColor = {
     FluoData: MuiColors.orange[500],
     SpikingActivityData: MuiColors.orange[500],
     BehaviorData: MuiColors.yellow[500],
+    MicroscopeData: MuiColors.purple[500],
   },
   nextKey: 0,
 }

diff --git a/frontend/src/store/slice/InputNode/InputNodeSelectors.ts b/frontend/src/store/slice/InputNode/InputNodeSelectors.ts
@@ -3,6 +3,7 @@ import {
   isCsvInputNode,
   isImageInputNode,
   isMatlabInputNode,
+  isMicroscopeInputNode,
 } from "store/slice/InputNode/InputNodeUtils"
 import { RootState } from "store/store"
 
@@ -62,6 +63,16 @@ export const selectMatlabInputNodeSelectedFilePath =
     }
   }
 
+export const selectMicroscopeInputNodeSelectedFilePath =
+  (nodeId: string) => (state: RootState) => {
+    const node = selectInputNodeById(nodeId)(state)
+    if (isMicroscopeInputNode(node)) {
+      return node.selectedFilePath
+    } else {
+      throw new Error("invalid input node type")
+    }
+  }
+
 export const selectFilePathIsUndefined = (state: RootState) =>
   Object.keys(state.inputNode).length === 0 ||
   Object.values(state.inputNode).filter((inputNode) => {

diff --git a/frontend/src/store/slice/InputNode/InputNodeSlice.ts b/frontend/src/store/slice/InputNode/InputNodeSlice.ts
@@ -144,6 +144,12 @@ export const inputNodeSlice = createSlice({
                 param: {},
               }
               break
+            case FILE_TYPE_SET.MICROSCOPE:
+              state[node.id] = {
+                fileType,
+                param: {},
+              }
+              break
           }
         }
       })
@@ -199,6 +205,11 @@ export const inputNodeSlice = createSlice({
                   fileType: FILE_TYPE_SET.HDF5,
                   param: {},
                 }
+              } else if (node.data.fileType === FILE_TYPE_SET.MICROSCOPE) {
+                newState[node.id] = {
+                  fileType: FILE_TYPE_SET.MICROSCOPE,
+                  param: {},
+                }
               }
             }
           })
@@ -238,6 +249,12 @@ export const inputNodeSlice = createSlice({
                     selectedFilePath: node.data.path as string,
                     param: {},
                   }
+                } else if (node.data.fileType === FILE_TYPE_SET.MICROSCOPE) {
+                  newState[node.id] = {
+                    fileType: FILE_TYPE_SET.MICROSCOPE,
+                    selectedFilePath: node.data.path as string,
+                    param: {},
+                  }
                 }
               }
             })

diff --git a/frontend/src/store/slice/InputNode/InputNodeType.ts b/frontend/src/store/slice/InputNode/InputNodeType.ts
@@ -7,6 +7,7 @@ export const FILE_TYPE_SET = {
   FLUO: "fluo",
   BEHAVIOR: "behavior",
   MATLAB: "matlab",
+  MICROSCOPE: "microscope",
 } as const
 
 export type FILE_TYPE = (typeof FILE_TYPE_SET)[keyof typeof FILE_TYPE_SET]
@@ -20,6 +21,7 @@ export type InputNodeType =
   | ImageInputNode
   | HDF5InputNode
   | MatlabInputNode
+  | MicroscopeInputNode
 
 interface InputNodeBaseType<
   T extends FILE_TYPE,
@@ -57,3 +59,8 @@ export interface HDF5InputNode
   selectedFilePath?: string
   hdf5Path?: string
 }
+
+export interface MicroscopeInputNode
+  extends InputNodeBaseType<"microscope", Record<never, never>> {
+  selectedFilePath?: string
+}
diff --git a/frontend/src/store/slice/InputNode/InputNodeUtils.ts b/frontend/src/store/slice/InputNode/InputNodeUtils.ts
@@ -5,6 +5,7 @@ import {
   InputNodeType,
   FILE_TYPE_SET,
   MatlabInputNode,
+  MicroscopeInputNode,
 } from "store/slice/InputNode/InputNodeType"
 
 export function isImageInputNode(
@@ -30,3 +31,9 @@ export function isHDF5InputNode(
 ): inputNode is HDF5InputNode {
   return inputNode.fileType === FILE_TYPE_SET.HDF5
 }
+
+export function isMicroscopeInputNode(
+  inputNode: InputNodeType,
+): inputNode is MicroscopeInputNode {
+  return inputNode.fileType === FILE_TYPE_SET.MICROSCOPE
+}
diff --git a/studio/app/Snakefile b/studio/app/Snakefile
@@ -17,7 +17,14 @@ else:
 
 
     for rule_name, details in config["rules"].items():
-        if details["type"] in [FILETYPE.IMAGE, FILETYPE.CSV, FILETYPE.BEHAVIOR, FILETYPE.HDF5, FILETYPE.MATLAB]:
+        if details["type"] in [
+            FILETYPE.IMAGE,
+            FILETYPE.CSV,
+            FILETYPE.BEHAVIOR,
+            FILETYPE.HDF5,
+            FILETYPE.MATLAB,
+            FILETYPE.MICROSCOPE,
+        ]:
             rule:
                 input:
                     SmkUtils.input(details)

diff --git a/studio/app/common/core/rules/data.py b/studio/app/common/core/rules/data.py
@@ -25,6 +25,7 @@
         FILETYPE.BEHAVIOR,
         FILETYPE.HDF5,
         FILETYPE.MATLAB,
+        FILETYPE.MICROSCOPE,
     ]:
         rule_config.input = snakemake.input[0]
 
@@ -42,3 +43,6 @@
     elif rule_config.type == FILETYPE.MATLAB:
         outputfile = FileWriter.mat(rule_config)
         PickleWriter.write(rule_config.output, outputfile)
+    elif rule_config.type == FILETYPE.MICROSCOPE:
+        outputfile = FileWriter.microscope(rule_config)
+        PickleWriter.write(rule_config.output, outputfile)
diff --git a/studio/app/common/core/rules/file_writer.py b/studio/app/common/core/rules/file_writer.py
@@ -5,6 +5,7 @@
 from studio.app.const import FILETYPE
 from studio.app.optinist.core.nwb.nwb import NWBDATASET
 from studio.app.optinist.dataclass.iscell import IscellData
+from studio.app.optinist.dataclass.microscope import MicroscopeData
 from studio.app.optinist.routers.mat import MatGetter
 
 
@@ -58,6 +59,13 @@ def mat(cls, rule_config: Rule):
         data = MatGetter.data(rule_config.input, rule_config.matPath)
         return cls.get_info_from_array_data(rule_config, nwbfile, data)
 
+    @classmethod
+    def microscope(cls, rule_config: Rule):
+        nwbfile = rule_config.nwbfile
+        info = {rule_config.return_arg: MicroscopeData(rule_config.input)}
+        info["nwbfile"] = {"input": nwbfile}
+        return info
+
     @classmethod
     def get_info_from_array_data(cls, rule_config: Rule, nwbfile, data):
         if data.ndim == 3:

diff --git a/studio/app/common/core/snakemake/smk_utils.py b/studio/app/common/core/snakemake/smk_utils.py
@@ -17,6 +17,7 @@ def input(cls, details):
             FILETYPE.BEHAVIOR,
             FILETYPE.HDF5,
             FILETYPE.MATLAB,
+            FILETYPE.MICROSCOPE,
         ]:
             return join_filepath([DIRPATH.INPUT_DIR, details["input"]])
         else:
@@ -34,6 +35,7 @@ def conda(cls, details):
             FILETYPE.BEHAVIOR,
             FILETYPE.HDF5,
             FILETYPE.MATLAB,
+            FILETYPE.MICROSCOPE,
         ]:
             return None
 

diff --git a/studio/app/common/core/snakemake/snakemake_rule.py b/studio/app/common/core/snakemake/snakemake_rule.py
@@ -61,6 +61,9 @@ def mat(self) -> Rule:
             .build()
         )
 
+    def microscope(self) -> Rule:
+        return self.builder.set_type(FILETYPE.MICROSCOPE).build()
+
     def algo(self, nodeDict: Dict[str, Node]) -> Rule:
         algo_input = []
         return_arg_names = {}

diff --git a/studio/app/common/core/workflow/workflow.py b/studio/app/common/core/workflow/workflow.py
@@ -14,6 +14,7 @@ class NodeType:
     BEHAVIOR: str = "BehaviorFileNode"
     HDF5: str = "HDF5FileNode"
     MAT: str = "MatlabFileNode"
+    MICROSCOPE: str = "MicroscopeFileNode"
     ALGO: str = "AlgorithmNode"
 
 

diff --git a/studio/app/common/core/workflow/workflow_result.py b/studio/app/common/core/workflow/workflow_result.py
@@ -46,10 +46,15 @@ def get(self, nodeIdList):
                         message=error_message,
                     )
 
-            glob_pickle_filepath = join_filepath(
-                [self.workflow_dirpath, node_id, "[!tmp_]*.pkl"]
+            glob_pickle_filepath = glob(
+                join_filepath([self.workflow_dirpath, node_id, "*.pkl"])
             )
-            for pickle_filepath in glob(glob_pickle_filepath):
+            tmp_glob_pickle_filepath = glob(
+                join_filepath([self.workflow_dirpath, node_id, "tmp_*.pkl"])
+            )
+            for pickle_filepath in list(
+                set(glob_pickle_filepath) - set(tmp_glob_pickle_filepath)
+            ):
                 results[node_id] = NodeResult(
                     self.workflow_dirpath,
                     node_id,

diff --git a/studio/app/common/core/workflow/workflow_runner.py b/studio/app/common/core/workflow/workflow_runner.py
@@ -126,6 +126,14 @@ def rulefile(self):
                     edgeDict=self.edgeDict,
                     nwbfile=nwbfile,
                 ).mat()
+            elif node.type == NodeType.MICROSCOPE:
+                rule_dict[node.id] = SmkRule(
+                    workspace_id=self.workspace_id,
+                    unique_id=self.unique_id,
+                    node=node,
+                    edgeDict=self.edgeDict,
+                    nwbfile=nwbfile,
+                ).microscope()
             elif node.type == NodeType.ALGO:
                 rule = SmkRule(
                     workspace_id=self.workspace_id,

diff --git a/studio/app/common/dataclass/image.py b/studio/app/common/dataclass/image.py
@@ -62,7 +62,7 @@ def save_json(self, json_dir):
 
     @property
     def output_path(self) -> OutputPath:
-        if self.data.ndim == 3:
+        if self.data.ndim >= 3:
             # self.path will be a list if self.data got into else statement on __init__
             if isinstance(self.path, list) and isinstance(self.path[0], str):
                 _path = self.path[0]