From 78c52ae73ba767fb69076d37a911b4c215e9ef19 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Mon, 17 Jun 2024 19:14:23 +0000 Subject: [PATCH] fixed bug in setting flags --- clients/python/utils/bathy_runner.py | 5 +++-- .../oceaneyes/atl24uncertainty/runner.py | 2 +- .../containers/oceaneyes/cshelph/runner.py | 9 +++++--- .../oceaneyes/medianfilter/runner.py | 8 ++++--- .../containers/oceaneyes/qtrees/combiner.py | 21 ++++++++++--------- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/clients/python/utils/bathy_runner.py b/clients/python/utils/bathy_runner.py index 9f7fe977f..0b424cfaf 100644 --- a/clients/python/utils/bathy_runner.py +++ b/clients/python/utils/bathy_runner.py @@ -41,9 +41,10 @@ # Build Output File Path (and credentials if necessary) if args.output_file == None: - output_filename = "/tmp/" + args.granule.replace("ATL03", "ATL24").replace(".h5", args.output_format) + output_filename = "/tmp/" + args.granule.replace("ATL03", "ATL24").replace(".h5", "." + args.output_format) + credentials = None elif args.output_file == "stage": - output_filename = "s3://sliderule/data/ATL24/" + args.granule.replace("ATL03", "ATL24").replace(".h5", args.output_format) + output_filename = "s3://sliderule/data/ATL24/" + args.granule.replace("ATL03", "ATL24").replace(".h5", "." + args.output_format) home_directory = os.path.expanduser('~') aws_credential_file = os.path.join(home_directory, '.aws', 'credentials') config = configparser.RawConfigParser() diff --git a/plugins/icesat2/containers/oceaneyes/atl24uncertainty/runner.py b/plugins/icesat2/containers/oceaneyes/atl24uncertainty/runner.py index 61357590f..f849d9117 100644 --- a/plugins/icesat2/containers/oceaneyes/atl24uncertainty/runner.py +++ b/plugins/icesat2/containers/oceaneyes/atl24uncertainty/runner.py @@ -215,7 +215,7 @@ def find_kd_range(kd_490): print("Calculating processing flags") data["max_sensor_depth"] = 1.8 / data["kd_490"] data["flags"] = 0 -data.loc[subaqueous & (data["kd_490"] > 0) & (data["depth"] > data["max_sensor_depth"])] = 1 +data.loc[subaqueous & (data["kd_490"] > 0) & (data["depth"] > data["max_sensor_depth"]), "flags"] = 1 ################ # WRITE OUTPUTS diff --git a/plugins/icesat2/containers/oceaneyes/cshelph/runner.py b/plugins/icesat2/containers/oceaneyes/cshelph/runner.py index 0f651f474..4de6c3515 100644 --- a/plugins/icesat2/containers/oceaneyes/cshelph/runner.py +++ b/plugins/icesat2/containers/oceaneyes/cshelph/runner.py @@ -104,9 +104,12 @@ min_buffer=min_buffer, max_buffer=max_buffer, sea_surface_label=41, bathymetry_label=40) -point_cloud['class_ph'] = results['classification'] + +df = pd.DataFrame() +df["index_ph"] = results['index_ph'] +df["class_ph"] = results['classification'] if output_csv != None: - point_cloud.to_csv(output_csv, index=False, columns=["index_ph", "class_ph"]) + df.to_csv(output_csv, index=False, columns=["index_ph", "class_ph"]) else: - print(point_cloud.to_string(index=False, header=True, columns=['index_ph', 'class_ph'])) + print(df.to_string(index=False, header=True, columns=['index_ph', 'class_ph'])) diff --git a/plugins/icesat2/containers/oceaneyes/medianfilter/runner.py b/plugins/icesat2/containers/oceaneyes/medianfilter/runner.py index 60228b387..7f70de691 100644 --- a/plugins/icesat2/containers/oceaneyes/medianfilter/runner.py +++ b/plugins/icesat2/containers/oceaneyes/medianfilter/runner.py @@ -104,9 +104,11 @@ segment_length=segment_length, compress_heights=compress_heights, compress_lats=compress_lats) -point_cloud['class_ph'] = results['classification'] +df = pd.DataFrame() +df["index_ph"] = results['index_ph'] +df["class_ph"] = results['classification'] if output_csv != None: - point_cloud.to_csv(output_csv, index=False, columns=["index_ph", "class_ph"]) + df.to_csv(output_csv, index=False, columns=["index_ph", "class_ph"]) else: - print(point_cloud.to_string(index=False, header=True, columns=['index_ph', 'class_ph'])) + print(df.to_string(index=False, header=True, columns=['index_ph', 'class_ph'])) diff --git a/plugins/icesat2/containers/oceaneyes/qtrees/combiner.py b/plugins/icesat2/containers/oceaneyes/qtrees/combiner.py index 67b78b77a..c9d89b580 100755 --- a/plugins/icesat2/containers/oceaneyes/qtrees/combiner.py +++ b/plugins/icesat2/containers/oceaneyes/qtrees/combiner.py @@ -43,19 +43,20 @@ qtrees_csv_file = sys.argv[2] # read in data +spot_df = pd.read_csv(spot_csv_file) qtrees_df = pd.read_csv(qtrees_csv_file) print("Read all into data frame") +# merge qtrees predictions into spot dataframe +trimmed_qtrees_df = pd.DataFrame() +trimmed_qtrees_df["index_ph"] = qtrees_df["index_ph"] +trimmed_qtrees_df["prediction"] = qtrees_df["prediction"] +spot_df = pd.merge(spot_df, trimmed_qtrees_df, on="index_ph", how='left') +del spot_df["prediction"] +spot_df.rename(columns={"prediction": "class_ph"}, inplace=True) + # write out new qtrees file -new_qtrees_df = pd.DataFrame() -new_qtrees_df["index_ph"] = qtrees_df["index_ph"] -new_qtrees_df["class_ph"] = qtrees_df["prediction"] -new_qtrees_df.to_csv(qtrees_csv_file, index=False) +trimmed_qtrees_df.to_csv(qtrees_csv_file, index=False) # write out new spot file -qtrees_df["class_ph"] = qtrees_df["prediction"] -qtrees_df["surface_h"] = qtrees_df["sea_surface_h"] -del qtrees_df["prediction"] -del qtrees_df["sea_surface_h"] -del qtrees_df["bathy_h"] -qtrees_df.to_csv(spot_csv_file, index=False) +spot_df.to_csv(spot_csv_file, index=False)