Refactored all fetch_s3 functions into one utility function

process_report/invoices/NERC_total_invoice.py

@@ -30,7 +30,11 @@ class NERCTotalInvoice(invoice.Invoice):
         invoice.SU_HOURS_FIELD,
         invoice.SU_TYPE_FIELD,
         invoice.RATE_FIELD,
+        invoice.GROUP_NAME_FIELD,
+        invoice.GROUP_INSTITUTION_FIELD,
+        invoice.GROUP_BALANCE_FIELD,
         invoice.COST_FIELD,
+        invoice.GROUP_BALANCE_USED_FIELD,
         invoice.CREDIT_FIELD,
         invoice.CREDIT_CODE_FIELD,
         invoice.BALANCE_FIELD,

process_report/invoices/billable_invoice.py

@@ -31,7 +31,11 @@ class BillableInvoice(invoice.Invoice):
         invoice.SU_HOURS_FIELD,
         invoice.SU_TYPE_FIELD,
         invoice.RATE_FIELD,
+        invoice.GROUP_NAME_FIELD,
+        invoice.GROUP_INSTITUTION_FIELD,
+        invoice.GROUP_BALANCE_FIELD,
         invoice.COST_FIELD,
+        invoice.GROUP_BALANCE_USED_FIELD,
         invoice.CREDIT_FIELD,
         invoice.CREDIT_CODE_FIELD,
         invoice.BALANCE_FIELD,

process_report/invoices/bu_internal_invoice.py

@@ -15,7 +15,11 @@ class BUInternalInvoice(invoice.Invoice):
         invoice.INVOICE_DATE_FIELD,
         invoice.PI_FIELD,
         "Project",
+        invoice.GROUP_NAME_FIELD,
+        invoice.GROUP_INSTITUTION_FIELD,
+        invoice.GROUP_BALANCE_FIELD,
         invoice.COST_FIELD,
+        invoice.GROUP_BALANCE_USED_FIELD,
         invoice.CREDIT_FIELD,
         invoice.SUBSIDY_FIELD,
         invoice.PI_BALANCE_FIELD,

process_report/invoices/invoice.py

@@ -12,6 +12,18 @@
 PI_2ND_USED = "2nd Month Used"
 ###
 
+### Prepay files fields
+PREPAY_MONTH_FIELD = "Month"
+PREPAY_CREDIT_FIELD = "Credit"
+PREPAY_DEBIT_FIELD = "Debit"
+PREPAY_GROUP_NAME_FIELD = "Group Name"
+PREPAY_GROUP_CONTACT_FIELD = "Group Contact Email"
+PREPAY_MANAGED_FIELD = "MGHPCC Managed"
+PREPAY_PROJECT_FIELD = "Project"
+PREPAY_START_DATE_FIELD = "Start Date"
+PREPAY_END_DATE_FIELD = "End Date"
+###
+
 ### Invoice field names
 INVOICE_DATE_FIELD = "Invoice Month"
 PROJECT_FIELD = "Project - Allocation"
@@ -21,6 +33,10 @@
 INVOICE_ADDRESS_FIELD = "Invoice Address"
 INSTITUTION_FIELD = "Institution"
 INSTITUTION_ID_FIELD = "Institution - Specific Code"
+GROUP_NAME_FIELD = "Prepaid Group Name"
+GROUP_INSTITUTION_FIELD = "Prepaid Group Institution"
+GROUP_BALANCE_FIELD = "Prepaid Group Balance"
+GROUP_BALANCE_USED_FIELD = "Prepaid Group Used"
 SU_HOURS_FIELD = "SU Hours (GBhr or SUhr)"
 SU_TYPE_FIELD = "SU Type"
 SU_CHARGE_FIELD = "SU Charge"
@@ -38,6 +54,7 @@
 MISSING_PI_FIELD = "Missing PI"
 PI_BALANCE_FIELD = "PI Balance"
 PROJECT_NAME_FIELD = "Project"
+GROUP_MANAGED_FIELD = "MGHPCC Managed"
 ###
 
 

process_report/invoices/pi_specific_invoice.py

@@ -27,7 +27,11 @@ class PIInvoice(invoice.Invoice):
         invoice.SU_HOURS_FIELD,
         invoice.SU_TYPE_FIELD,
         invoice.RATE_FIELD,
+        invoice.GROUP_NAME_FIELD,
+        invoice.GROUP_INSTITUTION_FIELD,
+        invoice.GROUP_BALANCE_FIELD,
         invoice.COST_FIELD,
+        invoice.GROUP_BALANCE_USED_FIELD,
         invoice.CREDIT_FIELD,
         invoice.CREDIT_CODE_FIELD,
         invoice.BALANCE_FIELD,

process_report/process_report.py

@@ -23,6 +23,7 @@
     validate_billable_pi_processor,
     new_pi_credit_processor,
     bu_subsidy_processor,
+    prepayment_processor,
 )
 
 ### PI file field names
@@ -54,8 +55,8 @@
 ###
 
 PI_S3_FILEPATH = "PIs/PI.csv"
-
 ALIAS_S3_FILEPATH = "PIs/alias.csv"
+PREPAY_DEBITS_S3_FILEPATH = "Prepay/prepay_debits.csv"
 
 
 logger = logging.getLogger(__name__)
@@ -77,6 +78,14 @@ def load_alias(alias_file):
     return alias_dict
 
 
+def load_prepay_csv(prepay_credits_path, prepay_projects_path, prepay_contacts_path):
+    return (
+        pandas.read_csv(prepay_credits_path),
+        pandas.read_csv(prepay_projects_path),
+        pandas.read_csv(prepay_contacts_path),
+    )
+
+
 def get_iso8601_time():
     return datetime.datetime.now().strftime("%Y%m%dT%H%M%SZ")
 
@@ -121,6 +130,24 @@ def main():
         required=True,
         help="File containing list of projects that are non-billable within a specified duration",
     )
+    parser.add_argument(
+        "--prepay-credits",
+        required=False,
+        default="prepaid_credits.csv",
+        help="CSV listing all prepay group credits. Defaults to 'prepaid_credits.csv'",
+    )
+    parser.add_argument(
+        "--prepay-projects",
+        required=False,
+        default="prepaid_projects.csv",
+        help="CSV listing all prepay group projects. Defaults to 'prepaid_projects.csv'",
+    )
+    parser.add_argument(
+        "--prepay-contacts",
+        required=False,
+        default="prepaid_contacts.csv",
+        help="CSV listing all prepay group contact information. Defaults to 'prepaid_contacts.csv'",
+    )
 
     parser.add_argument(
         "--nonbillable-file",
@@ -168,6 +195,11 @@ def main():
         required=False,
         help="Name of alias file listing PIs with aliases (and their aliases). If not provided, defaults to fetching from S3",
     )
+    parser.add_argument(
+        "--prepay-debits",
+        required=False,
+        help="Name of csv file listing all prepay group debits. If not provided, defaults to fetching from S3",
+    )
     parser.add_argument(
         "--BU-subsidy-amount",
         required=True,
@@ -186,14 +218,23 @@ def main():
     if args.old_pi_file:
         old_pi_file = args.old_pi_file
     else:
-        old_pi_file = fetch_s3_old_pi_file()
+        old_pi_file = util.fetch_s3(PI_S3_FILEPATH)
 
     if args.alias_file:
         alias_file = args.alias_file
     else:
-        alias_file = fetch_s3_alias_file()
+        alias_file = util.fetch_s3(ALIAS_S3_FILEPATH)
     alias_dict = load_alias(alias_file)
 
+    if args.prepay_debits:
+        prepay_debits_filepath = args.prepay_debits
+    else:
+        prepay_debits_filepath = util.fetch_s3(PREPAY_DEBITS_S3_FILEPATH)
+
+    prepay_credits, prepay_projects, prepay_info = load_prepay_csv(
+        args.prepay_credits, args.prepay_projects, args.prepay_contacts
+    )
+
     merged_dataframe = merge_csv(csv_files)
 
     pi = []
@@ -255,7 +296,19 @@ def main():
     )
     bu_subsidy_proc.process()
 
-    processed_data = bu_subsidy_proc.data
+    prepayment_proc = prepayment_processor.PrepaymentProcessor(
+        "",
+        invoice_month,
+        bu_subsidy_proc.data,
+        prepay_credits,
+        prepay_projects,
+        prepay_info,
+        prepay_debits_filepath,
+        args.upload_to_s3,
+    )
+    prepayment_proc.process()
+
+    processed_data = prepayment_proc.data
 
     ### Initialize invoices
 
@@ -371,20 +424,6 @@ def timed_projects(timed_projects_file, invoice_date):
     return dataframe[mask]["Project"].to_list()
 
 
-def fetch_s3_alias_file():
-    local_name = "alias.csv"
-    invoice_bucket = util.get_invoice_bucket()
-    invoice_bucket.download_file(ALIAS_S3_FILEPATH, local_name)
-    return local_name
-
-
-def fetch_s3_old_pi_file():
-    local_name = "PI.csv"
-    invoice_bucket = util.get_invoice_bucket()
-    invoice_bucket.download_file(PI_S3_FILEPATH, local_name)
-    return local_name
-
-
 def backup_to_s3_old_pi_file(old_pi_file):
     invoice_bucket = util.get_invoice_bucket()
     invoice_bucket.upload_file(old_pi_file, f"PIs/Archive/PI {get_iso8601_time()}.csv")

process_report/processors/add_institution_processor.py

@@ -14,28 +14,6 @@
 
 @dataclass
 class AddInstitutionProcessor(processor.Processor):
-    @staticmethod
-    def _get_institute_mapping(institute_list: list):
-        institute_map = dict()
-        for institute_info in institute_list:
-            for domain in institute_info["domains"]:
-                institute_map[domain] = institute_info["display_name"]
-
-        return institute_map
-
-    @staticmethod
-    def _get_institution_from_pi(institute_map, pi_uname):
-        institution_domain = pi_uname.split("@")[-1]
-        for i in range(institution_domain.count(".") + 1):
-            if institution_name := institute_map.get(institution_domain, ""):
-                break
-            institution_domain = institution_domain[institution_domain.find(".") + 1 :]
-
-        if institution_name == "":
-            logger.warning(f"PI name {pi_uname} does not match any institution!")
-
-        return institution_name
-
     def _add_institution(self):
         """Determine every PI's institution name, logging any PI whose institution cannot be determined
         This is performed by `get_institution_from_pi()`, which tries to match the PI's username to
@@ -49,7 +27,7 @@ def _add_institution(self):
         The list of mappings are defined in `institute_map.json`.
         """
         institute_list = util.load_institute_list()
-        institute_map = self._get_institute_mapping(institute_list)
+        institute_map = util.get_institute_mapping(institute_list)
         self.data = self.data.astype({invoice.INSTITUTION_FIELD: "str"})
         for i, row in self.data.iterrows():
             pi_name = row[invoice.PI_FIELD]
@@ -58,7 +36,7 @@ def _add_institution(self):
             else:
                 self.data.at[
                     i, invoice.INSTITUTION_FIELD
-                ] = self._get_institution_from_pi(institute_map, pi_name)
+                ] = util.get_institution_from_pi(institute_map, pi_name)
 
     def _process(self):
         self._add_institution()