From 6932ae0c1ec6eb1950df73ea97f61f53e1f415c3 Mon Sep 17 00:00:00 2001 From: Tom Barbette Date: Fri, 10 May 2024 09:02:50 +0200 Subject: [PATCH] Revert moving out export_pandas as it is now used by web --- npf/grapher.py | 1 - npf/pipeline/export_pandas.py | 63 ----------------------------------- npf_compare.py | 4 --- npf_run.py | 3 -- 4 files changed, 71 deletions(-) delete mode 100644 npf/pipeline/export_pandas.py diff --git a/npf/grapher.py b/npf/grapher.py index b933bcf..3a87147 100644 --- a/npf/grapher.py +++ b/npf/grapher.py @@ -154,7 +154,6 @@ def __init__(self, fname): def search(self, map_v): return next((v for k, v in self.items() if re.search(k,str(map_v))), None) - return next((v for k, v in self.items() if re.search(k,str(map_v))), None) def guess_type(d): diff --git a/npf/pipeline/export_pandas.py b/npf/pipeline/export_pandas.py deleted file mode 100644 index b12b1b5..0000000 --- a/npf/pipeline/export_pandas.py +++ /dev/null @@ -1,63 +0,0 @@ -import os - -import pandas as pd - -from npf.types.series import Series - -def export_pandas(options, series:Series, fileprefix:str = None) -> None: - # Add series to a pandas dataframe - if options.pandas_filename is None: - return - - #In pandas, we cant merge dataframe if two columns have the same name, which can happen - # if some RESULT have the same name than the variables. - # if that happens, we'll rename the result as y_XXX and add the name in overlapping - all_X=pd.DataFrame() # Empty dataframe for X - all_y=pd.DataFrame() # Empty dataframe for y - for _, build, all_results in series: - for i, x in enumerate(all_results): - try: - - labels = [k[1] if type(k) is tuple else k for k,v in x.read_variables().items()] - x_vars = [[v[1] if type(v) is tuple else v for k,v in x.read_variables().items()]] - x_vars=pd.DataFrame(x_vars,index=[0],columns=labels) - x_vars=pd.concat([pd.DataFrame({'build' :build.pretty_name()},index=[0]), pd.DataFrame({'test_index' :i},index=[0]), x_vars],axis=1) - y_target=pd.DataFrame.from_dict(all_results[x],orient='index').transpose() #Use orient='index' to handle lists with different lengths - - if len(y_target) == 0: - continue - - x_vars = pd.concat([x_vars]*len(y_target), ignore_index=True) - all_X = pd.concat([all_X,x_vars],ignore_index = True, axis=0) - y_target['run_index']=y_target.index - all_y = pd.concat([all_y,y_target],ignore_index = True, axis=0) - - - except Exception as e: - print(f"ERROR: When trying to export serie {build.pretty_name()}:") - raise(e) - - - if dup_columns := set(all_X.columns).intersection(set(all_y.columns)): - print("WARNING: The following outputs are also variable names. The columns for the output will be named with a y_ prefix.", dup_columns) - all_y = all_y.rename(columns=dict([(c, f'y_{c}') for c in dup_columns])) - - - try: - all_results_df = pd.concat([all_X, all_y],axis=1) - except Exception as e: - print("ERROR: When trying to concatenate variables and features") - print(all_X) - print(all_y) - raise(e) - - # Save the pandas dataframe into a csv - pandas_df_name = (os.path.splitext(options.pandas_filename)[0] + - (f"-{fileprefix}" if fileprefix else "") + ".csv") - # Create the destination folder if it doesn't exist - df_path = os.path.dirname(pandas_df_name) - if df_path and not os.path.exists(df_path): - os.makedirs(df_path) - - all_results_df.to_csv(pandas_df_name, index=True, index_label="index", sep=",", header=True) - print(f"Pandas dataframe written to {pandas_df_name}") \ No newline at end of file diff --git a/npf_compare.py b/npf_compare.py index d4bdfca..2aaf309 100755 --- a/npf_compare.py +++ b/npf_compare.py @@ -10,8 +10,6 @@ import argparse from typing import Dict from npf.pipeline import pypost -from npf.pipeline import export_pandas - from npf.regression import * @@ -73,7 +71,6 @@ def export_output(filename: str, series: Series , kind_series:Series,options): return pypost.execute_pypost(series) - export_pandas.export_pandas(options, series) if options.do_time: print(kind_series) @@ -81,7 +78,6 @@ def export_output(filename: str, series: Series , kind_series:Series,options): for kind, dataset in kind_dataset.items(): try: pypost.execute_pypost(dataset) - export_pandas.export_pandas(options, dataset, fileprefix=kind) except Exception as e: print(f"While exporting dataset for kind {kind}:") print(dataset) diff --git a/npf_run.py b/npf_run.py index 0207af2..158e6dc 100755 --- a/npf_run.py +++ b/npf_run.py @@ -9,7 +9,6 @@ from npf import npf from npf.pipeline import pypost -from npf.pipeline import export_pandas from npf.regression import * from npf.statistics import Statistics from npf.test import Test, ScriptInitException @@ -267,7 +266,6 @@ def main(): filename = args.graph_filename or build.result_path(test.filename, 'pdf') series_with_history = [(test, build, all_results)] + g_series pypost.execute_pypost(series=series_with_history) - export_pandas.export_pandas(options=args, series= series_with_history) grapher.graph(series=series_with_history, title=test.get_title(), filename=filename, @@ -279,7 +277,6 @@ def main(): continue series_with_history = [(test, build, results)] pypost.execute_pypost(series=series_with_history) - export_pandas.export_pandas(options=args, series= series_with_history, fileprefix=kind) grapher.graph(series=series_with_history, title=test.get_title(), filename=filename,