From 46b431b641a6e746abb835c3874a119b0071ef14 Mon Sep 17 00:00:00 2001 From: gferraro Date: Thu, 7 Nov 2024 18:10:18 +0100 Subject: [PATCH] add multiple segment type option --- src/build.py | 3 +- src/ml_tools/dataset.py | 46 +--- src/ml_tools/datasetstructures.py | 343 ++++++++++++++++-------------- src/ml_tools/hyperparams.py | 16 +- src/ml_tools/interpreter.py | 3 +- src/ml_tools/kerasmodel.py | 1 - src/ml_tools/tfwriter.py | 31 +-- src/ml_tools/thermalwriter.py | 2 +- src/ml_tools/tools.py | 4 +- src/modelevaluate.py | 8 +- src/track/track.py | 38 ++-- 11 files changed, 226 insertions(+), 269 deletions(-) diff --git a/src/build.py b/src/build.py index ef416f2c..3c766af5 100644 --- a/src/build.py +++ b/src/build.py @@ -896,7 +896,7 @@ def main(): { "segment_frame_spacing": master_dataset.segment_spacing * 9, "segment_width": master_dataset.segment_length, - "segment_type": master_dataset.segment_type, + "segment_types": master_dataset.segment_types, "segment_min_avg_mass": master_dataset.segment_min_avg_mass, "max_segments": master_dataset.max_segments, "dont_filter_segment": True, @@ -932,6 +932,7 @@ def main(): "counts": dataset_counts, "by_label": False, "config": attrs.asdict(config), + "segment_types": master_dataset.segment_types, } with open(meta_filename, "w") as f: diff --git a/src/ml_tools/dataset.py b/src/ml_tools/dataset.py index 8556c5f9..7e633b32 100644 --- a/src/ml_tools/dataset.py +++ b/src/ml_tools/dataset.py @@ -83,7 +83,7 @@ def __init__( self.excluded_tags = config.build.excluded_tags self.min_frame_mass = config.build.min_frame_mass self.filter_by_lq = config.build.filter_by_lq - self.segment_type = SegmentType.ALL_RANDOM + self.segment_types = [SegmentType.ALL_RANDOM] self.max_segments = config.build.max_segments self.country = config.build.country self.max_frames = config.build.max_frames @@ -100,7 +100,7 @@ def __init__( self.segment_spacing = 1 self.segment_min_avg_mass = 10 self.min_frame_mass = 16 - self.segment_type = SegmentType.ALL_RANDOM + self.segment_types = [SegmentType.ALL_RANDOM] self.max_frames = 75 self.country_rectangle = BuildConfig.COUNTRY_LOCATIONS.get(self.country) @@ -244,7 +244,7 @@ def load_clip(self, db_clip, dont_filter_segment=False): track_header.get_segments( segment_width, segment_frame_spacing, - self.segment_type, + self.segment_types, self.segment_min_avg_mass, max_segments=self.max_segments, dont_filter=dont_filter_segment, @@ -504,46 +504,6 @@ def regroup( def has_data(self): return len(self.samples_by_id) > 0 - # - # def recalculate_segments(self, segment_type=SegmentType.ALL_RANDOM): - # self.samples_by_bin.clear() - # self.samples_by_label.clear() - # del self.samples[:] - # del self.samples - # self.samples = [] - # self.samples_by_label = {} - # self.samples_by_bin = {} - # logging.info("%s generating segments type %s", self.name, segment_type) - # start = time.time() - # empty_tracks = [] - # filtered_stats = 0 - # - # for track in self.tracks: - # segment_frame_spacing = int( - # round(self.segment_spacing * track.frames_per_second) - # ) - # segment_width = self.segment_length - # track.calculate_segments( - # segment_frame_spacing, - # segment_width, - # segment_type, - # segment_min_mass=segment_min_avg_mass, - # ) - # filtered_stats = filtered_stats + track.filtered_stats["segment_mass"] - # if len(track.segments) == 0: - # empty_tracks.append(track) - # continue - # for sample in track.segments: - # self.add_clip_sample_mappings(sample) - # - # self.rebuild_cdf() - # logging.info( - # "%s #segments %s filtered stats are %s took %s", - # self.name, - # len(self.samples), - # filtered_stats, - # time.time() - start, - # ) def remove_sample_by_id(self, id, bin_id): del self.samples_by_id[id] try: diff --git a/src/ml_tools/datasetstructures.py b/src/ml_tools/datasetstructures.py index 6a7fbd61..daa41741 100644 --- a/src/ml_tools/datasetstructures.py +++ b/src/ml_tools/datasetstructures.py @@ -367,7 +367,7 @@ def get_segments( self, segment_width, segment_frame_spacing=9, - segment_type=SegmentType.ALL_RANDOM, + segment_types=[SegmentType.ALL_RANDOM], segment_min_mass=None, repeats=1, max_segments=None, @@ -389,7 +389,7 @@ def get_segments( # in python3.7+ can just take the values and it guarantees order it was added to dict regions = self.bounds_history - self.samples, self.filtered_stats = get_segments( + self.samples, filtered_stats = get_segments( self.clip_id, self.track_id, self.start_frame, @@ -402,7 +402,7 @@ def get_segments( lower_mass=self.lower_mass, repeats=repeats, min_frames=min_frames, - segment_type=segment_type, + segment_types=segment_types, max_segments=max_segments, station_id=self.station_id, source_file=self.source_file, @@ -412,6 +412,7 @@ def get_segments( fp_frames=self.fp_frames if filter_by_fp else None, rec_time=self.start_time, ) + self.filtered_stats.update(filtered_stats) # GP could get this from the tracks when writing # but might be best to keep samples independent for ease for s in self.samples: @@ -974,8 +975,7 @@ def get_segments( lower_mass=0, repeats=1, min_frames=None, - segment_frames=None, - segment_type=SegmentType.ALL_RANDOM, + segment_types=[SegmentType.ALL_RANDOM], max_segments=None, location=None, station_id=None, @@ -986,9 +986,8 @@ def get_segments( skip_ffc=True, frame_min_mass=None, fp_frames=None, + repeat_frame_indices=True, ): - if segment_type == SegmentType.ALL_RANDOM_NOMIN: - segment_min_mass = None if min_frames is None: min_frames = segment_width / 4.0 segments = [] @@ -997,163 +996,189 @@ def get_segments( has_no_mass = np.sum(mass_history) == 0 - frame_indices = [ - region.frame_number - for region in regions - if (has_no_mass or region.mass > 0) - and ( - ffc_frames is None - or skip_ffc is False - or region.frame_number not in ffc_frames - ) - and not region.blank - and region.width > 0 - and region.height > 0 - and ((has_no_mass or frame_min_mass is None) or region.mass >= frame_min_mass) - ] - if fp_frames is not None and label not in FP_LABELS: - frame_indices = [f for f in frame_indices if f not in fp_frames] - if len(frame_indices) == 0: - logging.warn("Nothing to load for %s - %s", clip_id, track_id) - return [], filtered_stats - if segment_min_mass is not None: - segment_min_mass = min( - segment_min_mass, - np.median(mass_history[frame_indices - start_frame]), - ) - else: - segment_min_mass = 1 - # remove blank frames - - if segment_type == SegmentType.TOP_RANDOM: - # take top 50 mass frames - frame_indices = sorted( - frame_indices, - key=lambda f_i: mass_history[f_i - start_frame], - reverse=True, - ) - frame_indices = frame_indices[:50] - frame_indices.sort() - if segment_type == SegmentType.TOP_SEQUENTIAL: - return get_top_mass_segments( - clip_id, - track_id, - label, - camera, - segment_width, - segment_frame_spacing, - mass_history, - ffc_frames, - regions, - start_frame, - lower_mass, - segment_min_mass, - source_file=source_file, - ) - if len(frame_indices) < min_frames: - filtered_stats["too short"] += 1 - return segments, filtered_stats - frame_indices = np.array(frame_indices) - segment_count = max(1, len(frame_indices) // segment_frame_spacing) - segment_count = int(segment_count) - if max_segments is not None: - segment_count = min(max_segments, segment_count) - - # take any segment_width frames, this could be done each epoch - whole_indices = frame_indices - random_frames = segment_type in [ - SegmentType.IMPORTANT_RANDOM, - SegmentType.ALL_RANDOM, - SegmentType.ALL_RANDOM_NOMIN, - SegmentType.TOP_RANDOM, - None, - ] - for _ in range(repeats): - frame_indices = whole_indices.copy() - if random_frames: - # random_frames and not random_sections: - np.random.shuffle(frame_indices) - for i in range(segment_count): - # always get atleast one segment, not doing annymore - if (len(frame_indices) < segment_width / 2.0 and len(segments) > 1) or len( - frame_indices - ) < segment_width / 4: - break + for segment_type in segment_types: + s_min_mass = segment_min_mass + if segment_type == SegmentType.ALL_RANDOM_NOMIN: + s_min_mass = None - if segment_type == SegmentType.ALL_SECTIONS: - # random frames from section 2.2 * segment_width - section = frame_indices[: int(segment_width * 2.2)] - indices = np.random.choice( - len(section), - min(segment_width, len(section)), - replace=False, - ) - frames = section[indices] - # might need to change that gp 11/05 - 2024 - frame_indices = frame_indices[segment_frame_spacing:] - elif random_frames: - # frame indices already randomized so just need to grab some - frames = frame_indices[:segment_width] - frame_indices = frame_indices[segment_width:] - else: - segment_start = i * segment_frame_spacing - segment_end = segment_start + segment_width - segment_end = min(len(frame_indices), segment_end) - frames = frame_indices[segment_start:segment_end] - - remaining = segment_width - len(frames) - # sample another same frames again if need be - if remaining > 0: - extra_frames = np.random.choice( - frames, - min(remaining, len(frames)), - replace=False, - ) - frames = np.concatenate([frames, extra_frames]) - frames.sort() - relative_frames = frames - start_frame - mass_slice = mass_history[relative_frames] - segment_mass = np.sum(mass_slice) - segment_avg_mass = segment_mass / len(mass_slice) - filtered = False - if segment_min_mass and segment_avg_mass < segment_min_mass: - if dont_filter: - filtered = True - else: - filtered_stats["segment_mass"] += 1 - continue - - # temp_slice = frame_temp_median[relative_frames] - region_slice = regions[relative_frames] - movement_data = None - if segment_avg_mass < 50: - segment_weight_factor = 0.75 - elif segment_avg_mass < 100: - segment_weight_factor = 1 - else: - segment_weight_factor = 1.2 - - for z, f in enumerate(frames): - assert region_slice[z].frame_number == f - segment = SegmentHeader( + frame_indices = [ + region.frame_number + for region in regions + if (has_no_mass or region.mass > 0) + and ( + ffc_frames is None + or skip_ffc is False + or region.frame_number not in ffc_frames + ) + and not region.blank + and region.width > 0 + and region.height > 0 + and ( + (has_no_mass or frame_min_mass is None) or region.mass >= frame_min_mass + ) + ] + if fp_frames is not None and label not in FP_LABELS: + frame_indices = [f for f in frame_indices if f not in fp_frames] + if len(frame_indices) == 0: + logging.warn("Nothing to load for %s - %s", clip_id, track_id) + return [], filtered_stats + if s_min_mass is not None: + s_min_mass = min( + s_min_mass, + np.median(mass_history[frame_indices - start_frame]), + ) + else: + s_min_mass = 1 + # remove blank frames + + if segment_type == SegmentType.TOP_RANDOM: + # take top 50 mass frames + frame_indices = sorted( + frame_indices, + key=lambda f_i: mass_history[f_i - start_frame], + reverse=True, + ) + frame_indices = frame_indices[:50] + frame_indices.sort() + if segment_type == SegmentType.TOP_SEQUENTIAL: + new_segments, filtered = get_top_mass_segments( clip_id, track_id, - start_frame=start_frame, - frames=segment_width, - weight=segment_weight_factor, - mass=segment_mass, - label=label, - regions=region_slice, - frame_indices=frames, - movement_data=movement_data, - camera=camera, - location=location, - station_id=station_id, - rec_time=rec_time, + label, + camera, + segment_width, + segment_frame_spacing, + mass_history, + ffc_frames, + regions, + start_frame, + lower_mass, + s_min_mass, source_file=source_file, - filtered=filtered, ) - segments.append(segment) + segments.extend(new_segments) + filtered_stats.merge(filtered) + continue + if len(frame_indices) < min_frames: + filtered_stats["too short"] += 1 + continue + + frame_indices = np.array(frame_indices) + segment_count = max(1, len(frame_indices) // segment_frame_spacing) + segment_count = int(segment_count) + # probably only counts for all random + if max_segments is not None and segment_type not in [SegmentType.ALL_SECTIONS]: + segment_count = min(max_segments, segment_count) + + # take any segment_width frames, this could be done each epoch + whole_indices = frame_indices + random_frames = segment_type in [ + SegmentType.IMPORTANT_RANDOM, + SegmentType.ALL_RANDOM, + SegmentType.ALL_RANDOM_NOMIN, + SegmentType.TOP_RANDOM, + None, + ] + for _ in range(repeats): + frame_indices = whole_indices.copy() + if random_frames: + # random_frames and not random_sections: + np.random.shuffle(frame_indices) + for i in range(segment_count): + # always get atleast one segment, not doing annymore + if ( + len(frame_indices) < segment_width / 2.0 and len(segments) > 1 + ) or len(frame_indices) < segment_width / 4: + break + + if segment_type == SegmentType.ALL_SECTIONS: + # random frames from section 2.2 * segment_width + section = frame_indices[: int(segment_width * 2.2)] + + indices = np.random.choice( + len(section), + min(segment_width, len(section)), + replace=False, + ) + frames = section[indices] + # might need to change that gp 11/05 - 2024 + frame_indices = frame_indices[segment_width:] + elif random_frames: + # frame indices already randomized so just need to grab some + frames = frame_indices[:segment_width] + frame_indices = frame_indices[segment_width:] + else: + segment_start = i * segment_frame_spacing + segment_end = segment_start + segment_width + segment_end = min(len(frame_indices), segment_end) + frames = frame_indices[segment_start:segment_end] + + remaining = segment_width - len(frames) + # sample another same frames again if need be + if remaining > 0: + extra_frames = np.random.choice( + frames, + min(remaining, len(frames)), + replace=False, + ) + frames = np.concatenate([frames, extra_frames]) + frames.sort() + relative_frames = frames - start_frame + mass_slice = mass_history[relative_frames] + segment_mass = np.sum(mass_slice) + segment_avg_mass = segment_mass / len(mass_slice) + filtered = False + if s_min_mass and segment_avg_mass < s_min_mass: + if dont_filter: + filtered = True + else: + filtered_stats["segment_mass"] += 1 + continue + + # temp_slice = frame_temp_median[relative_frames] + region_slice = regions[relative_frames] + movement_data = None + if segment_avg_mass < 50: + segment_weight_factor = 0.75 + elif segment_avg_mass < 100: + segment_weight_factor = 1 + else: + segment_weight_factor = 1.2 + + for z, f in enumerate(frames): + assert region_slice[z].frame_number == f + + if repeat_frame_indices: + # i think this can be default, means we dont need to handle + # short segments elsewhere + if len(frames) < segment_width: + extra_samples = np.random.choice( + frames, segment_width - len(frames) + ) + frames = list(frames) + frames.extend(extra_samples) + frames.sort() + + segment = SegmentHeader( + clip_id, + track_id, + start_frame=start_frame, + frames=segment_width, + weight=segment_weight_factor, + mass=segment_mass, + label=label, + regions=region_slice, + frame_indices=frames, + movement_data=movement_data, + camera=camera, + location=location, + station_id=station_id, + rec_time=rec_time, + source_file=source_file, + filtered=filtered, + ) + segments.append(segment) + return segments, filtered_stats diff --git a/src/ml_tools/hyperparams.py b/src/ml_tools/hyperparams.py index cd6ddb79..b1868fd0 100644 --- a/src/ml_tools/hyperparams.py +++ b/src/ml_tools/hyperparams.py @@ -24,7 +24,7 @@ def insert_defaults(self): self["square_width"] = self.square_width self["frame_size"] = self.frame_size self["segment_width"] = self.segment_width - self["segment_type"] = self.segment_type + self["segment_types"] = self.segment_types self["multi_label"] = True self["diff_norm"] = self.diff_norm self["thermal_diff_norm"] = self.thermal_diff_norm @@ -89,12 +89,14 @@ def segment_width(self): return self.get("segment_width", 25 if self.use_segments else 1) @property - def segment_type(self): - segment_type = self.get("segment_type", SegmentType.ALL_RANDOM.name) - if isinstance(segment_type, str): - return SegmentType[segment_type] - else: - return segment_type + def segment_types(self): + + segment_types = self.get("segment_type", [SegmentType.ALL_RANDOM]) + # convert string to enum type + if isinstance(segment_types[0], str): + for i in range(len(segment_types)): + segment_types[i] = SegmentType[segment_types[i]] + return segment_types @property def mvm(self): diff --git a/src/ml_tools/interpreter.py b/src/ml_tools/interpreter.py index 2b299181..bdac4f53 100644 --- a/src/ml_tools/interpreter.py +++ b/src/ml_tools/interpreter.py @@ -21,6 +21,7 @@ def load_json(self, filename): self.version = metadata.get("version", None) self.labels = metadata["labels"] self.params = HyperParams() + print("Hypers are ", metadata.get("hyperparams", {})) self.params.update(metadata.get("hyperparams", {})) self.data_type = metadata.get("type", "thermal") @@ -298,7 +299,7 @@ def preprocess_segments( ffc_frames=[] if dont_filter else clip.ffc_frames, repeats=1, segment_frames=segment_frames, - segment_type=self.params.segment_type, + segment_types=self.params.segment_types, from_last=predict_from_last, max_segments=max_segments, dont_filter=dont_filter, diff --git a/src/ml_tools/kerasmodel.py b/src/ml_tools/kerasmodel.py index 4e5acf8a..45cbb466 100644 --- a/src/ml_tools/kerasmodel.py +++ b/src/ml_tools/kerasmodel.py @@ -1083,7 +1083,6 @@ def plot_confusion_matrix(cm, class_names): counts = cm.copy() threshold = counts.max() / 2.0 - print("Threshold is", threshold, " for ", cm.max()) # Normalize the confusion matrix. cm = np.around(cm.astype("float") / cm.sum(axis=1)[:, np.newaxis], decimals=2) diff --git a/src/ml_tools/tfwriter.py b/src/ml_tools/tfwriter.py index d40cf8ac..983308f2 100644 --- a/src/ml_tools/tfwriter.py +++ b/src/ml_tools/tfwriter.py @@ -12,31 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -from PIL import Image from pathlib import Path from multiprocessing import Process, Queue - -import collections -import hashlib -import io -import json -import multiprocessing import os -import time -from absl import app -from absl import flags from absl import logging import numpy as np -from PIL import Image, ImageOps - import tensorflow as tf -from . import tfrecord_util -from ml_tools import tools -from ml_tools.imageprocessing import normalize, rotate -from track.cliptracker import get_diff_back_filtered -import cv2 -import random -import math def process_job(queue, labels, base_dir, save_data, writer_i, extra_args): @@ -44,7 +25,6 @@ def process_job(queue, labels, base_dir, save_data, writer_i, extra_args): pid = os.getpid() - # writer_i = 1 name = f"{writer_i}-{pid}.tfrecord" logging.info("Writing to %s", name) options = tf.io.TFRecordOptions(compression_type="GZIP") @@ -66,15 +46,8 @@ def process_job(queue, labels, base_dir, save_data, writer_i, extra_args): saved += save_data(samples, writer, labels, extra_args) files += 1 del samples - # if saved > 250000 / num_frames: - # logging.info("Closing old writer") - # writer.close() - # writer_i += 1 - # name = f"{writer_i}-{pid}.tfrecord" - # logging.info("Opening %s", name) - # saved = 0 - # writer = tf.io.TFRecordWriter(str(base_dir / name), options=options) - if i % int(25000 / num_frames) == 0: + + if i % int(2500 / num_frames) == 0: logging.info("Saved %s ", files) gc.collect() writer.flush() diff --git a/src/ml_tools/thermalwriter.py b/src/ml_tools/thermalwriter.py index 7a123460..891edb54 100644 --- a/src/ml_tools/thermalwriter.py +++ b/src/ml_tools/thermalwriter.py @@ -227,7 +227,7 @@ def get_data(clip_samples, extra_args): segment_frame_spacing=extra_args.get( "segment_frame_spacing", 9 ), - segment_type=extra_args.get("segment_type"), + segment_types=extra_args.get("segment_types"), segment_min_mass=extra_args.get("segment_min_avg_mass"), dont_filter=extra_args.get("dont_filter_segment", False), skip_ffc=extra_args.get("skip_ffc", True), diff --git a/src/ml_tools/tools.py b/src/ml_tools/tools.py index 38dd9e90..ce604906 100644 --- a/src/ml_tools/tools.py +++ b/src/ml_tools/tools.py @@ -15,6 +15,7 @@ from pathlib import Path from ml_tools.rectangle import Rectangle from dateutil import parser +from enum import Enum EPISON = 1e-5 @@ -54,7 +55,8 @@ def default(self, obj): return obj.meta_dictionary() elif isinstance(obj, Path): return str(obj) - + elif isinstance(obj, Enum): + return str(obj.name) # Let the base class default method raise the TypeError return json.JSONEncoder.default(self, obj) diff --git a/src/modelevaluate.py b/src/modelevaluate.py index 94ae85e9..fb07124f 100644 --- a/src/modelevaluate.py +++ b/src/modelevaluate.py @@ -463,14 +463,14 @@ def evaluate_dir( masses = np.array(data[4]) masses = masses[:, None] top_score = None - # if model.params.multi_label is True: - # # every label could be 1 for each prediction - # top_score = len(output) + if model.params.multi_label is True: + # # every label could be 1 for each prediction + top_score = np.sum(masses) # smoothed = output # else: smoothed = output * masses prediction.classified_clip( - output, smoothed, data[2], top_score=top_score + output, smoothed, data[2], masses, top_score=top_score ) y_true.append(label_mapping.get(label, label)) predicted_labels = [prediction.predicted_tag()] diff --git a/src/track/track.py b/src/track/track.py index 165ee39d..b8264c35 100644 --- a/src/track/track.py +++ b/src/track/track.py @@ -439,7 +439,7 @@ def get_segments( repeats=1, min_frames=0, segment_frames=None, - segment_type=SegmentType.ALL_RANDOM, + segment_types=[SegmentType.ALL_RANDOM], from_last=None, max_segments=None, ffc_frames=None, @@ -477,28 +477,22 @@ def get_segments( ) segments.append(segment) else: - all_segments = [] - for seg_type in [SegmentType.ALL_RANDOM, SegmentType.ALL_SECTIONS]: - segments, _ = get_segments( - self.clip_id, - self._id, - start_frame, - segment_frame_spacing=segment_frame_spacing, - segment_width=segment_width, - regions=regions, - ffc_frames=ffc_frames, - repeats=repeats, - # frame_temp_median=frame_temp_median, - min_frames=min_frames, - segment_frames=None, - segment_type=seg_type, - max_segments=max_segments, - dont_filter=dont_filter, - # segment_type=seg_type, - ) - all_segments.extend(segments) + segments, _ = get_segments( + self.clip_id, + self._id, + start_frame, + segment_frame_spacing=segment_frame_spacing, + segment_width=segment_width, + regions=regions, + ffc_frames=ffc_frames, + repeats=repeats, + min_frames=min_frames, + segment_types=segment_types, + max_segments=max_segments, + dont_filter=dont_filter, + ) - return all_segments + return segments @classmethod def from_region(cls, clip, region, tracker_version=None, tracking_config=None):