From 31082f70a8f90a37f4b4c2c5f2aa33cef378a20e Mon Sep 17 00:00:00 2001 From: Dave McKay Date: Tue, 10 Sep 2024 14:58:59 +0100 Subject: [PATCH] clean-up --- csd3-side/scripts/Untitled-1.ipynb | 212 ----------------------------- 1 file changed, 212 deletions(-) delete mode 100644 csd3-side/scripts/Untitled-1.ipynb diff --git a/csd3-side/scripts/Untitled-1.ipynb b/csd3-side/scripts/Untitled-1.ipynb deleted file mode 100644 index 83761d2..0000000 --- a/csd3-side/scripts/Untitled-1.ipynb +++ /dev/null @@ -1,212 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "folders = ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data']\n", - "folder_files = [['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o60432140', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.e61815332', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e60432140', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.e61815344', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.61815332', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.o61815344', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.61815344', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/data_folders_20220504.txt', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e5514523', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e5533764', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/trace', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.5514523', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/data_folders_20220505.txt', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o5514523', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/terabyte_folders_20210708.txt', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.60432140', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.5533764', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o5533764', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.sh', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.o61815332'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20210322.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20200820.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview_20210409.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview_20201102.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20210315.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20220526.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20210331.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/viking_images_overview_20201218.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20210315.csv.old', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_tiles_tracts_patches_overview_20200820.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/viking_images_overview_20210406.csv']]\n", - "num_zips = 9" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "36" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sum([len(files) for files in folder_files])" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "chunk_subfolders = False\n", - "if num_zips > len(folders):\n", - " chunk_subfolders = True" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "chunk_subfolders" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "in: chunking subfolders\n", - "folders: ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data']\n", - "folder_files: [['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o60432140', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.e61815332', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e60432140', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.e61815344', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.61815332', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.o61815344', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.61815344', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/data_folders_20220504.txt', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e5514523', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e5533764', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/trace', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.5514523', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/data_folders_20220505.txt', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o5514523', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/terabyte_folders_20210708.txt', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.60432140', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.5533764', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o5533764', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.sh', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.o61815332'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20210322.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20200820.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview_20210409.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview_20201102.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20210315.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20220526.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20210331.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/viking_images_overview_20201218.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20210315.csv.old', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_tiles_tracts_patches_overview_20200820.csv', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/viking_images_overview_20210406.csv']]\n", - "num_zips: 9\n", - "j: 0\n", - "22\n", - "2\n", - "i: 0\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o60432140', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.e61815332']\n", - "i: 2\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e60432140', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.e61815344']\n", - "i: 4\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.61815332', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.o61815344']\n", - "i: 6\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.61815344', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/data_folders_20220504.txt']\n", - "i: 8\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e5514523']\n", - "i: 10\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e5533764']\n", - "i: 12\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/trace', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.5514523']\n", - "i: 14\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/data_folders_20220505.txt', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o5514523']\n", - "i: 16\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/terabyte_folders_20210708.txt', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.60432140']\n", - "i: 18\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.5533764', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o5533764']\n", - "i: 20\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.sh', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.o61815332']\n", - "j: 1\n", - "14\n", - "1\n", - "i: 0\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20210322.csv']\n", - "i: 1\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20200820.csv']\n", - "i: 2\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview.csv']\n", - "i: 3\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview_20210409.csv']\n", - "i: 4\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview_20201102.csv']\n", - "i: 5\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20210315.csv']\n", - "i: 6\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview.csv']\n", - "i: 7\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview.csv']\n", - "i: 8\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20220526.csv']\n", - "i: 9\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20210331.csv']\n", - "i: 10\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/viking_images_overview_20201218.csv']\n", - "i: 11\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20210315.csv.old']\n", - "i: 12\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_tiles_tracts_patches_overview_20200820.csv']\n", - "i: 13\n", - "['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/viking_images_overview_20210406.csv']\n", - "subchunks_files: [['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o60432140', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.e61815332'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e60432140', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.e61815344'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.61815332', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.o61815344'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.61815344', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/data_folders_20220504.txt'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e5514523'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.e5533764'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/trace', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.5514523'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/data_folders_20220505.txt', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o5514523'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/terabyte_folders_20210708.txt', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.60432140'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/machine.file.5533764', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/2_Data_size.slurm.o5533764'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.sh', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm/1_Data_overview.slurm.o61815332'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20210322.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20200820.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview_20210409.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview_20201102.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20210315.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/vhs_images_overview.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20220526.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_images_overview_20210331.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/viking_images_overview_20201218.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/hsc_images_overview_20210315.csv.old'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/video_tiles_tracts_patches_overview_20200820.csv'], ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data/viking_images_overview_20210406.csv']]\n", - "subchunks: ['/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/slurm', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data', '/rds/project/rds-rPTGgs6He74/ras81/lsst-ir-fusion/dmu1/data']\n", - "50 25\n" - ] - } - ], - "source": [ - "if chunk_subfolders:\n", - " print('in: chunking subfolders')\n", - " print('folders:',folders)\n", - " print('folder_files:',folder_files)\n", - " print('num_zips:',num_zips)\n", - " subchunks_files = []\n", - " subchunks = []\n", - " for j in range(len(folders)):\n", - " print('j:',j)\n", - " print(len(folder_files[j]))\n", - " print(len(folder_files[j])//num_zips)\n", - " for i in range(0, len(folder_files[j]), len(folder_files[j])//num_zips):\n", - " print('i:',i)\n", - " print(folder_files[j][i:i+len(folder_files[j])//num_zips])\n", - " subchunks_files.append(folder_files[j][i:i+len(folder_files[j])//num_zips])\n", - " subchunks.append([folder for folder in folders[j]])\n", - " print('subchunks_files:',subchunks_files)\n", - " \n", - " print('subchunks:',subchunks)\n", - " print(len(subchunks), len(subchunks_files))" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(50, 25)" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(chunks), len(chunk_files)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "lsst-uk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}