Add missing fields. Logging vs printing

finance/management/commands/xformnetfilerawdata.py

@@ -5,6 +5,7 @@
 """
 
 import warnings
+import logging
 from dateutil.parser import parse as date_parse
 from itertools import izip_longest
 from numbers import Number
@@ -408,10 +409,10 @@ def load_form_row(row, agency, force=False, verbosity=1):  # noqa
         print row
         raise E
 
-    if verbosity:
-        if created:
-            print "created"
-        print(str(money))
+    c = ""
+    if created:
+        c = "created "
+    logging.debug("%s%s", c,  str(money))
 
     return money, True
 
@@ -495,9 +496,8 @@ def find_unloaded_rows(data, skip_rate=100, force=False, verbosity=1):
     for xacts in grouper(skip_rate, xact_keys):
         vals = [v['source_xact_id'] for v in models.IndependentMoney.objects.filter(
             source='NF', source_xact_id__in=xacts).values('source_xact_id')]
-        if verbosity:
-            for val in vals:
-                print("Skipping NF/%s" % val)
+        for val in vals:
+            logging.debug("Skipping NF/%s", val)
 
         if force:
             yield set(xacts)
@@ -519,6 +519,7 @@ def load_form_data(data, agency_fn, form_name, form_type=None,
     error_rows = []
     xact_key_generator = find_unloaded_rows(data, force=force, verbosity=verbosity)
     xact_keys = []
+    count = 0;
     for ri, (_, raw_row) in enumerate(data.T.iteritems()):
         # Quickly get near an unloaded row.
         while not xact_keys and xact_keys is not None:
@@ -548,6 +549,10 @@ def load_form_data(data, agency_fn, form_name, form_type=None,
             # TODO: Store errors, for review later.
             raise
 
+        count += 1
+        if (count % 1000) == 0:
+            print("Loaded %d records" % count)
+
     return error_rows
 
 

netfile_raw/migrations/0002_auto_20160810_0335.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('netfile_raw', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='netfilecal201transaction',
+            name='filerLocalId',
+            field=models.CharField(max_length=32, null=True, db_column='filerLocalId'),
+        ),
+        migrations.AddField(
+            model_name='netfilecal201transaction',
+            name='filerStateId',
+            field=models.CharField(max_length=2, null=True, db_column='filerStateId'),
+        ),
+        migrations.AddField(
+            model_name='netfilecal201transaction',
+            name='filingEndDate',
+            field=models.DateField(null=True, db_column='filingEndDate'),
+        ),
+        migrations.AddField(
+            model_name='netfilecal201transaction',
+            name='filingStartDate',
+            field=models.DateField(null=True, db_column='filingStartDate'),
+        ),
+    ]

netfile_raw/models.py

@@ -75,6 +75,9 @@ class NetFileCal201Transaction(CalAccessBaseModel):
     tran_NamT = models.CharField(
         max_length=256,
         db_column='tran_NamT')
+    filingStartDate = models.DateField(
+        null=True,
+        db_column='filingStartDate')
     tran_Dscr = models.CharField(
         max_length=256,
         db_column='tran_Dscr')
@@ -150,6 +153,9 @@ class NetFileCal201Transaction(CalAccessBaseModel):
     intr_NamF = models.CharField(
         max_length=256,
         db_column='intr_NamF')
+    filingEndDate = models.DateField(
+        null=True,
+        db_column='filingEndDate')
     transactionType = models.IntegerField(
         null=True,
         db_column='transactionType')
@@ -255,6 +261,10 @@ class NetFileCal201Transaction(CalAccessBaseModel):
     int_CmteId = models.CharField(
         max_length=32,
         db_column='int_CmteId')
+    filerLocalId = models.CharField(
+        null=True,
+        max_length=32,
+        db_column='filerLocalId')
     calculated_Date = models.DateField(
         null=True,
         db_column='calculated_Date')
@@ -288,6 +298,10 @@ class NetFileCal201Transaction(CalAccessBaseModel):
     entity_Cd = models.CharField(
         max_length=3,
         db_column='entity_Cd')
+    filerStateId = models.CharField(
+        max_length=2,
+        null=True,
+        db_column='filerStateId')
     tres_Adr1 = models.CharField(
         max_length=256,
         db_column='tres_Adr1')