Merge pull request #259 from kahst:raven

combined Raven selection table

analyze.py

@@ -47,7 +47,7 @@ def saveResultFile(r: dict[str, list], path: str, afile_path: str):
 
     if cfg.RESULT_TYPE == "table":
         # Raven selection header
-        header = "Selection\tView\tChannel\tBegin File\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies Code\tCommon Name\tConfidence\n"
+        header = "Selection\tView\tChannel\tBegin Path\tFile Duration (s)\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies Code\tCommon Name\tConfidence\n"
         selection_id = 0
         filename = os.path.basename(afile_path)
 
@@ -72,9 +72,10 @@ def saveResultFile(r: dict[str, list], path: str, afile_path: str):
                 if c[1] > cfg.MIN_CONFIDENCE and (not cfg.SPECIES_LIST or c[0] in cfg.SPECIES_LIST):
                     selection_id += 1
                     label = cfg.TRANSLATED_LABELS[cfg.LABELS.index(c[0])]
-                    rstring += "{}\tSpectrogram 1\t1\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{:.4f}\n".format(
+                    rstring += "{}\tSpectrogram 1\t1\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{:.4f}\n".format(
                         selection_id,
-                        filename,
+                        afile_path,
+                        audio.getAudioFileLength(afile_path, cfg.SAMPLE_RATE),
                         start,
                         end,
                         low_freq,
@@ -86,6 +87,23 @@ def saveResultFile(r: dict[str, list], path: str, afile_path: str):
 
             # Write result string to file
             out_string += rstring
+
+        # If we don't have any valid predictions, we still need to add a line to the selection table in case we want to combine results
+        # TODO: That's a weird way to do it, but it works for now. It would be better to keep track of file durations during the analysis.
+        if len(out_string) == len(header) and cfg.OUTPUT_PATH is not None:
+            selection_id += 1
+            out_string += "{}\tSpectrogram 1\t1\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{:.4f}\n".format(
+                selection_id,
+                afile_path,
+                audio.getAudioFileLength(afile_path, cfg.SAMPLE_RATE),
+                0,
+                3,
+                low_freq,
+                high_freq,
+                "nocall",
+                "nocall",
+                1.0,
+            )
 
     elif cfg.RESULT_TYPE == "audacity":
         # Audacity timeline labels
@@ -190,6 +208,61 @@ def saveResultFile(r: dict[str, list], path: str, afile_path: str):
     with open(path, "w", encoding="utf-8") as rfile:
         rfile.write(out_string)
 
+def combineResults(folder: str, output_file: str):
+
+    # Read all files
+    files = utils.collect_all_files(folder, "txt", pattern="BirdNET.selection.table")
+
+    # Combine all files  
+    s_id = 1
+    time_offset = 0
+    with open(os.path.join(folder, output_file), "w", encoding="utf-8") as f:
+        f.write("Selection\tView\tChannel\tBegin Path\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies Code\tCommon Name\tConfidence\n")
+        for rfile in files:
+            with open(rfile, "r", encoding="utf-8") as rf:
+
+                try:
+
+                    lines = rf.readlines()
+                    # make sure it's a selection table
+                    if not "Selection" in lines[0] or not "File Duration" in lines[0]:
+                        continue
+
+                    # skip header and add to file
+                    f_duration = float(lines[1].split("\t")[4])
+                    for line in lines[1:]:
+
+                        # empty line?
+                        if not line.strip():
+                            continue
+
+                        # Is species code and common name == 'nocall'?
+                        # If so, that's a dummy line and we can skip it
+                        if line.split("\t")[9] == "nocall" and line.split("\t")[10] == "nocall":
+                            continue
+
+                        # adjust selection id
+                        line = line.split("\t")
+                        line[0] = str(s_id)
+                        s_id += 1
+
+                        # adjust time
+                        line[5] = str(float(line[5]) + time_offset)
+                        line[6] = str(float(line[6]) + time_offset)
+
+                        # remove File Duration
+                        del line[4]
+
+                        # write line
+                        f.write("\t".join(line))
+
+                    # adjust time offset
+                    time_offset += f_duration     
+
+                except Exception as ex:
+                    print(f"Error: Cannot combine results from {rfile}.\n", flush=True)
+                    utils.writeErrorLog(ex)       
+
 
 def getSortedTimestamps(results: dict[str, list]):
     """Sorts the results based on the segments.
@@ -399,6 +472,11 @@ def analyzeFile(item):
         default="table",
         help="Specifies output format. Values in ['table', 'audacity', 'r',  'kaleidoscope', 'csv']. Defaults to 'table' (Raven selection table).",
     )
+    parser.add_argument(
+        "--output_file",
+        default=None,
+        help="Path to combined Raven selection table. If set and rtype is 'table', all results will be combined into this file. Defaults to None."
+    )
     parser.add_argument("--threads", type=int, default=multiprocessing.cpu_count() // 2, help="Number of CPU threads.")
     parser.add_argument(
         "--batchsize", type=int, default=1, help="Number of samples to process at the same time. Defaults to 1."
@@ -528,6 +606,12 @@ def analyzeFile(item):
     if not cfg.RESULT_TYPE in ["table", "audacity", "r", "kaleidoscope", "csv"]:
         cfg.RESULT_TYPE = "table"
 
+    # Set output file
+    if args.output_file is not None and cfg.RESULT_TYPE == "table":
+        cfg.OUTPUT_FILE = args.output_file
+    else:
+        cfg.OUTPUT_FILE = None
+
     # Set number of threads
     if os.path.isdir(cfg.INPUT_PATH):
         cfg.CPU_THREADS = max(1, int(args.threads))
@@ -551,7 +635,16 @@ def analyzeFile(item):
             analyzeFile(entry)
     else:
         with Pool(cfg.CPU_THREADS) as p:
-            p.map(analyzeFile, flist)
+            # Map analyzeFile function to each entry in flist
+            results = p.map_async(analyzeFile, flist)
+            # Wait for all tasks to complete
+            results.wait()            
+
+    # Combine results?
+    if not cfg.OUTPUT_FILE is None:
+        print("Combining results into {}...".format(cfg.OUTPUT_FILE), end='', flush=True)
+        combineResults(cfg.OUTPUT_PATH, cfg.OUTPUT_FILE)
+        print("done!", flush=True)
 
     # A few examples to test
     # python3 analyze.py --i example/ --o example/ --slist example/ --min_conf 0.5 --threads 4

config.py

@@ -107,6 +107,7 @@
 # 'audacity' denotes a TXT file with the same format as Audacity timeline labels
 # 'csv' denotes a generic CSV file with start, end, species and confidence.
 RESULT_TYPE: str = "table"
+OUTPUT_FILENAME: str = "BirdNET_SelectionTable.txt" # this is for combined Raven selection tables only
 
 #####################
 # Training settings #
@@ -228,6 +229,7 @@ def getConfig():
         'MIN_CONFIDENCE': MIN_CONFIDENCE,
         'BATCH_SIZE': BATCH_SIZE,
         'RESULT_TYPE': RESULT_TYPE,
+        'OUTPUT_FILENAME': OUTPUT_FILENAME,
         'TRAIN_DATA_PATH': TRAIN_DATA_PATH,
         'SAMPLE_CROP_MODE': SAMPLE_CROP_MODE,
         'NON_EVENT_CLASSES': NON_EVENT_CLASSES,
@@ -289,6 +291,7 @@ def setConfig(c):
     global MIN_CONFIDENCE
     global BATCH_SIZE
     global RESULT_TYPE
+    global OUTPUT_FILENAME
     global TRAIN_DATA_PATH
     global SAMPLE_CROP_MODE
     global NON_EVENT_CLASSES
@@ -346,6 +349,7 @@ def setConfig(c):
     MIN_CONFIDENCE = c['MIN_CONFIDENCE']
     BATCH_SIZE = c['BATCH_SIZE']
     RESULT_TYPE = c['RESULT_TYPE']
+    OUTPUT_FILENAME = c['OUTPUT_FILENAME']
     TRAIN_DATA_PATH = c['TRAIN_DATA_PATH']
     SAMPLE_CROP_MODE = c['SAMPLE_CROP_MODE']
     NON_EVENT_CLASSES = c['NON_EVENT_CLASSES']

gui.py

@@ -20,6 +20,7 @@
 import species
 import utils
 from train import trainModel
+import webbrowser
 
 _WINDOW: webview.Window
 OUTPUT_TYPE_MAP = {
@@ -105,6 +106,7 @@ def runSingleFileAnalysis(
         sf_thresh,
         custom_classifier_file,
         "csv",
+        None,
         "en" if not locale else locale,
         1,
         4,
@@ -129,6 +131,8 @@ def runBatchAnalysis(
     sf_thresh,
     custom_classifier_file,
     output_type,
+    output_filename,
+    combine_tables,
     locale,
     batch_size,
     threads,
@@ -159,6 +163,7 @@ def runBatchAnalysis(
         sf_thresh,
         custom_classifier_file,
         output_type,
+        output_filename if combine_tables else None,
         "en" if not locale else locale,
         batch_size if batch_size and batch_size > 0 else 1,
         threads if threads and threads > 0 else 4,
@@ -184,6 +189,7 @@ def runAnalysis(
     sf_thresh: float,
     custom_classifier_file,
     output_type: str,
+    output_filename: str | None,
     locale: str,
     batch_size: int,
     threads: int,
@@ -209,6 +215,7 @@ def runAnalysis(
         sf_thresh: The threshold for the predicted species list.
         custom_classifier_file: Custom classifier to be used.
         output_type: The type of result to be generated.
+        output_filename: The filename for the combined output.
         locale: The translation to be used.
         batch_size: The number of samples in a batch.
         threads: The number of threads to be used.
@@ -312,6 +319,12 @@ def runAnalysis(
     if not cfg.RESULT_TYPE in ["table", "audacity", "r", "csv"]:
         cfg.RESULT_TYPE = "table"
 
+    # Set output filename
+    if output_filename is not None and cfg.RESULT_TYPE == "table":
+        cfg.OUTPUT_FILE = output_filename
+    else:
+        cfg.OUTPUT_FILE = None
+
     # Set number of threads
     if input_dir:
         cfg.CPU_THREADS = max(1, int(threads))
@@ -349,6 +362,12 @@ def runAnalysis(
 
                 result_list.append(result)
 
+    # Combine results?
+    if not cfg.OUTPUT_FILE is None:
+        print("Combining results into {}...".format(cfg.OUTPUT_FILE), end='', flush=True)
+        analyze.combineResults(cfg.OUTPUT_PATH, cfg.OUTPUT_FILE)
+        print("done!", flush=True)
+
     return [[os.path.relpath(r[0], input_dir), r[1]] for r in result_list] if input_dir else cfg.OUTPUT_PATH
 
 
@@ -958,13 +977,45 @@ def select_directory_wrapper():
                 selected_classifier_state,
             ) = species_lists()
 
-            output_type_radio = gr.Radio(
-                list(OUTPUT_TYPE_MAP.keys()),
-                value="Raven selection table",
-                label="Result type",
-                info="Specifies output format.",
-            )
+            with gr.Accordion("Output type", open=True):
+
+                output_type_radio = gr.Radio(
+                    list(OUTPUT_TYPE_MAP.keys()),
+                    value="Raven selection table",
+                    label="Result type",
+                    info="Specifies output format.",
+                )
+
+                with gr.Row():
+                    with gr.Column():
+                        combine_tables_checkbox = gr.Checkbox(
+                            False,
+                            label="Combine selection tables",
+                            info="If checked, all selection tables are combined into one.",
+                        )
+
+                    with gr.Column():
+                        output_filename = gr.Textbox(
+                            "BirdNET_Results_Selection_Table.txt",
+                            label="Output filename",
+                            info="Name of the combined selection table.",
+                            visible=False,
+                        )
 
+                    def on_output_type_change(value, check):
+                        return gr.Checkbox(visible=value == "Raven selection table"), gr.Textbox(visible=check)
+
+                    output_type_radio.change(
+                        on_output_type_change, inputs=[output_type_radio, combine_tables_checkbox], outputs=[combine_tables_checkbox, output_filename], show_progress=False
+                    )
+
+                    def on_combine_tables_change(value):
+                        return gr.Textbox(visible=value)
+
+                    combine_tables_checkbox.change(
+                        on_combine_tables_change, inputs=combine_tables_checkbox, outputs=output_filename, show_progress=False
+                    )
+
             with gr.Row():
                 batch_size_number = gr.Number(
                     precision=1, label="Batch size", value=1, info="Number of samples to process at the same time."
@@ -993,6 +1044,8 @@ def select_directory_wrapper():
                 sf_thresh_number,
                 selected_classifier_state,
                 output_type_radio,
+                output_filename,
+                combine_tables_checkbox,
                 locale_radio,
                 batch_size_number,
                 threads_number,

utils.py

@@ -27,6 +27,26 @@ def collect_audio_files(path: str):
 
     return sorted(files)
 
+def collect_all_files(path: str, filetypes: list[str], pattern: str = ""):
+    """Collects all files of the given filetypes in the given directory.
+
+    Args:
+        path: The directory to be searched.
+        filetypes: A list of filetypes to be collected.
+
+    Returns:
+        A sorted list of all files in the directory.
+    """
+
+    files = []
+
+    for root, _, flist in os.walk(path):
+        for f in flist:
+            if not f.startswith(".") and f.rsplit(".", 1)[-1].lower() in filetypes and (pattern in f or pattern == ""):
+                files.append(os.path.join(root, f))
+
+    return sorted(files)
+
 
 def readLines(path: str):
     """Reads the lines into a list.