Merge pull request PSLmodels#327 from PSLmodels/pr-conform-cds-to-sta…

…te-preparation

Pr conform cds to state preparation

tmd/areas/targets/.gitignore

@@ -0,0 +1 @@
+*.csv

tmd/areas/targets/de00_targets.csv

@@ -0,0 +1,132 @@
+varname,count,scope,agilo,agihi,fstatus,target
+XTOT,0,0,-9e99,9e99,0,1003384
+c00100,0,1,-9e99,1,0,-248171000
+c00100,0,1,1,10000,0,266363000
+c00100,0,1,10000,25000,0,1441521000
+c00100,0,1,25000,50000,0,4265193000
+c00100,0,1,50000,75000,0,4468916000
+c00100,0,1,75000,100000,0,4261264000
+c00100,0,1,100000,200000,0,11534532000
+c00100,0,1,200000,500000,0,7713961000
+c00100,0,1,500000,9e99,0,7885870000
+c00100,1,1,-9e99,1,0,8760
+c00100,1,1,1,10000,0,51390
+c00100,1,1,10000,25000,0,83360
+c00100,1,1,25000,50000,0,116540
+c00100,1,1,50000,75000,0,72460
+c00100,1,1,75000,100000,0,49140
+c00100,1,1,100000,200000,0,83970
+c00100,1,1,200000,500000,0,27270
+c00100,1,1,500000,9e99,0,5560
+c00100,1,1,-9e99,1,1,6040
+c00100,1,1,1,10000,1,41800
+c00100,1,1,10000,25000,1,57670
+c00100,1,1,25000,50000,1,68390
+c00100,1,1,50000,75000,1,36790
+c00100,1,1,75000,100000,1,17960
+c00100,1,1,100000,200000,1,15610
+c00100,1,1,200000,500000,1,3430
+c00100,1,1,500000,9e99,1,790
+c00100,1,1,-9e99,1,2,1380
+c00100,1,1,1,10000,2,3490
+c00100,1,1,10000,25000,2,8690
+c00100,1,1,25000,50000,2,18470
+c00100,1,1,50000,75000,2,21340
+c00100,1,1,75000,100000,2,24180
+c00100,1,1,100000,200000,2,62620
+c00100,1,1,200000,500000,2,22890
+c00100,1,1,500000,9e99,2,4570
+c00100,1,1,-9e99,1,4,1090
+c00100,1,1,1,10000,4,5290
+c00100,1,1,10000,25000,4,15340
+c00100,1,1,25000,50000,4,25400
+c00100,1,1,50000,75000,4,11080
+c00100,1,1,75000,100000,4,5390
+c00100,1,1,100000,200000,4,4220
+c00100,1,1,200000,500000,4,680
+c00100,1,1,500000,9e99,4,130
+e00200,0,1,-9e99,1,0,40998000
+e00200,0,1,1,10000,0,202705000
+e00200,0,1,10000,25000,0,1005926000
+e00200,0,1,25000,50000,0,3310445000
+e00200,0,1,50000,75000,0,3219104000
+e00200,0,1,75000,100000,0,2867857000
+e00200,0,1,100000,200000,0,7597123000
+e00200,0,1,200000,500000,0,4707271000
+e00200,0,1,500000,9e99,0,2286246000
+e00200,2,1,-9e99,1,0,970
+e00200,2,1,1,10000,0,36290
+e00200,2,1,10000,25000,0,62370
+e00200,2,1,25000,50000,0,97560
+e00200,2,1,50000,75000,0,58830
+e00200,2,1,75000,100000,0,38970
+e00200,2,1,100000,200000,0,67820
+e00200,2,1,200000,500000,0,22880
+e00200,2,1,500000,9e99,0,4460
+e00300,0,1,-9e99,1,0,1875000
+e00300,0,1,1,10000,0,2624000
+e00300,0,1,10000,25000,0,6004000
+e00300,0,1,25000,50000,0,11530000
+e00300,0,1,50000,75000,0,14228000
+e00300,0,1,75000,100000,0,14094000
+e00300,0,1,100000,200000,0,44601000
+e00300,0,1,200000,500000,0,40283000
+e00300,0,1,500000,9e99,0,64159000
+e01500,0,1,-9e99,1,0,18739193.64931253
+e01500,0,1,1,10000,0,49614008.58630007
+e01500,0,1,10000,25000,0,206430475.4088788
+e01500,0,1,25000,50000,0,518851237.2031721
+e01500,0,1,50000,75000,0,712666623.9202255
+e01500,0,1,75000,100000,0,846928700.7538186
+e01500,0,1,100000,200000,0,2450983803.628015
+e01500,0,1,200000,500000,0,1422257603.1295624
+e01500,0,1,500000,9e99,0,419222470.0539074
+e02400,0,1,1,10000,0,354663427.17321765
+e02400,0,1,10000,25000,0,465438989.1312245
+e02400,0,1,25000,50000,0,535116684.79032177
+e02400,0,1,50000,75000,0,469115729.03045046
+e02400,0,1,75000,100000,0,435957820.28787345
+e02400,0,1,100000,200000,0,902892648.1447308
+e02400,0,1,200000,500000,0,280920876.66106105
+e02400,0,1,500000,9e99,0,53486260.97354113
+e18400,0,1,1,10000,0,1610517.9481968267
+e18400,0,1,10000,25000,0,3214389.013831858
+e18400,0,1,25000,50000,0,20903915.11158256
+e18400,0,1,50000,75000,0,55626167.11692388
+e18400,0,1,75000,100000,0,88446957.69953683
+e18400,0,1,100000,200000,0,339778870.1096634
+e18400,0,1,200000,500000,0,311831381.6737225
+e18400,0,1,500000,9e99,0,392455776.21674454
+e18400,2,1,1,10000,0,818.2778007797441
+e18400,2,1,10000,25000,0,2696.020216663356
+e18400,2,1,25000,50000,0,12723.65315766416
+e18400,2,1,50000,75000,0,22040.452432942984
+e18400,2,1,75000,100000,0,24014.490363521167
+e18400,2,1,100000,200000,0,53891.344394331885
+e18400,2,1,200000,500000,0,21160.8713526379
+e18400,2,1,500000,9e99,0,4111.853811014874
+e18500,0,1,1,10000,0,2760909.911359199
+e18500,0,1,10000,25000,0,7196621.134468874
+e18500,0,1,25000,50000,0,21057233.78434795
+e18500,0,1,50000,75000,0,40538373.89106983
+e18500,0,1,75000,100000,0,48859123.240578435
+e18500,0,1,100000,200000,0,143472167.56916553
+e18500,0,1,200000,500000,0,89787378.42109823
+e18500,0,1,500000,9e99,0,44491412.883680046
+e18500,2,1,1,10000,0,1077.3370921879753
+e18500,2,1,10000,25000,0,3062.1176738521176
+e18500,2,1,25000,50000,0,10456.720239339402
+e18500,2,1,50000,75000,0,19267.513032546423
+e18500,2,1,75000,100000,0,21430.484792377032
+e18500,2,1,100000,200000,0,49096.64913686987
+e18500,2,1,200000,500000,0,19989.154486175117
+e18500,2,1,500000,9e99,0,4663.028133886486
+e26270,0,1,-9e99,1,0,-75581000
+e26270,0,1,1,10000,0,-4409000
+e26270,0,1,10000,25000,0,6000
+e26270,0,1,25000,50000,0,11520000
+e26270,0,1,50000,75000,0,21536000
+e26270,0,1,75000,100000,0,29599000
+e26270,0,1,100000,200000,0,167784000
+e26270,0,1,200000,500000,0,438995000
+e26270,0,1,500000,9e99,0,1951914000

tmd/areas/targets/ny21_targets.csv

@@ -1,5 +1,6 @@
 varname,count,scope,agilo,agihi,fstatus,target
 XTOT,0,0,-9e99,9e99,0,778806.9963618957
+c00100,0,1,-9e99,1,0,-211051658.59288508
 c00100,0,1,1,10000,0,182423342.11336407
 c00100,0,1,10000,25000,0,1037335916.1426524
 c00100,0,1,25000,50000,0,3369386206.2984276
@@ -8,6 +9,42 @@ c00100,0,1,75000,100000,0,3192887737.1752
 c00100,0,1,100000,200000,0,7444391217.009996
 c00100,0,1,200000,500000,0,3533125242.646242
 c00100,0,1,500000,9e99,0,3512409391.634545
+c00100,1,1,-9e99,1,0,7783.30061591544
+c00100,1,1,1,10000,0,34845.00467650245
+c00100,1,1,10000,25000,0,58872.665215289446
+c00100,1,1,25000,50000,0,92073.22923991714
+c00100,1,1,50000,75000,0,57122.387347592645
+c00100,1,1,75000,100000,0,36860.296870916754
+c00100,1,1,100000,200000,0,55175.53757452468
+c00100,1,1,200000,500000,0,12676.61287259154
+c00100,1,1,500000,9e99,0,2380.3685403844206
+c00100,1,1,-9e99,1,1,5025.5683979292535
+c00100,1,1,1,10000,1,28500.37375066762
+c00100,1,1,10000,25000,1,40999.78237965522
+c00100,1,1,25000,50000,1,55431.39047078707
+c00100,1,1,50000,75000,1,27828.250423235033
+c00100,1,1,75000,100000,1,11313.918480070117
+c00100,1,1,100000,200000,1,8421.1356665729
+c00100,1,1,200000,500000,1,1666.754799870258
+c00100,1,1,500000,9e99,1,375.3116805477707
+c00100,1,1,-9e99,1,2,1474.965149593628
+c00100,1,1,1,10000,2,3227.4938250458044
+c00100,1,1,10000,25000,2,7784.831857302671
+c00100,1,1,25000,50000,2,17620.627308577044
+c00100,1,1,50000,75000,2,20112.27469996074
+c00100,1,1,75000,100000,2,21629.717785468103
+c00100,1,1,100000,200000,2,43900.36435388441
+c00100,1,1,200000,500000,2,10580.306783344742
+c00100,1,1,500000,9e99,2,1903.525763605093
+c00100,1,1,-9e99,1,4,1041.8053348183391
+c00100,1,1,1,10000,4,2545.6814968580734
+c00100,1,1,10000,25000,4,8850.43355733606
+c00100,1,1,25000,50000,4,16010.696051542369
+c00100,1,1,50000,75000,4,6907.72695791437
+c00100,1,1,75000,100000,4,2875.8964103872336
+c00100,1,1,100000,200000,4,1963.7019740835344
+c00100,1,1,200000,500000,4,297.62913704836967
+c00100,1,1,500000,9e99,4,61.76571232759558
 e00200,0,1,-9e99,1,0,28535896.06329254
 e00200,0,1,1,10000,0,132417077.07366145
 e00200,0,1,10000,25000,0,620330137.749753
@@ -17,31 +54,76 @@ e00200,0,1,75000,100000,0,2158790311.823775
 e00200,0,1,100000,200000,0,5030364975.228401
 e00200,0,1,200000,500000,0,2069653304.213072
 e00200,0,1,500000,9e99,0,1092633770.975998
+e00200,2,1,-9e99,1,0,806.7386660978011
+e00200,2,1,1,10000,0,23984.276637414303
+e00200,2,1,10000,25000,0,40408.03541532223
+e00200,2,1,25000,50000,0,76433.55188506725
+e00200,2,1,50000,75000,0,46844.03447107544
+e00200,2,1,75000,100000,0,29958.328880574227
+e00200,2,1,100000,200000,0,46038.66418516202
+e00200,2,1,200000,500000,0,10454.521354775869
+e00200,2,1,500000,9e99,0,1945.450019463003
+e00300,0,1,-9e99,1,0,1581441.3343929157
+e00300,0,1,1,10000,0,2059511.4825179589
+e00300,0,1,10000,25000,0,5453454.860042885
+e00300,0,1,25000,50000,0,8987752.705774691
+e00300,0,1,50000,75000,0,10385728.858139034
+e00300,0,1,75000,100000,0,10498788.919467432
+e00300,0,1,100000,200000,0,28513365.955123864
+e00300,0,1,200000,500000,0,20078380.506733365
+e00300,0,1,500000,9e99,0,35057765.47688019
+e01500,0,1,-9e99,1,0,22307334.01281969
+e01500,0,1,1,10000,0,47848592.034152426
+e01500,0,1,10000,25000,0,187708091.3319156
+e01500,0,1,25000,50000,0,461572027.44619393
+e01500,0,1,50000,75000,0,626554897.4175295
+e01500,0,1,75000,100000,0,686820779.8501711
+e01500,0,1,100000,200000,0,1673877373.614286
+e01500,0,1,200000,500000,0,664624676.8486961
+e01500,0,1,500000,9e99,0,162652022.7591892
+e02400,0,1,1,10000,0,171298809.01726258
+e02400,0,1,10000,25000,0,317693726.0086798
+e02400,0,1,25000,50000,0,413944478.5434576
+e02400,0,1,50000,75000,0,365059932.9543816
+e02400,0,1,75000,100000,0,314140783.0287367
+e02400,0,1,100000,200000,0,521367504.32092035
+e02400,0,1,200000,500000,0,127240255.18700288
+e02400,0,1,500000,9e99,0,23150322.73235176
+e18400,0,1,1,10000,0,1011802.43810994
+e18400,0,1,10000,25000,0,1656831.1519416494
+e18400,0,1,25000,50000,0,9942329.342997376
+e18400,0,1,50000,75000,0,23798183.94379118
+e18400,0,1,75000,100000,0,33988931.48938802
+e18400,0,1,100000,200000,0,111446722.3570964
+e18400,0,1,200000,500000,0,102380449.1237176
+e18400,0,1,500000,9e99,0,224371672.05052748
+e18400,2,1,1,10000,0,865.5489383820963
+e18400,2,1,10000,25000,0,1935.834861330082
+e18400,2,1,25000,50000,0,6137.9549070856565
+e18400,2,1,50000,75000,0,9926.483531215375
+e18400,2,1,75000,100000,0,9800.234664955417
+e18400,2,1,100000,200000,0,18994.51441407621
+e18400,2,1,200000,500000,0,6666.388826266086
+e18400,2,1,500000,9e99,0,1489.7200290137812
+e18500,0,1,1,10000,0,3359548.199909812
+e18500,0,1,10000,25000,0,8307917.037332345
+e18500,0,1,25000,50000,0,21631077.55065034
+e18500,0,1,50000,75000,0,39285321.10971896
+e18500,0,1,75000,100000,0,42273138.739311695
+e18500,0,1,100000,200000,0,101295518.30666286
+e18500,0,1,200000,500000,0,58314192.13054088
+e18500,0,1,500000,9e99,0,33548786.640014283
+e18500,2,1,1,10000,0,649.1153612422925
+e18500,2,1,10000,25000,0,1626.6553974119277
+e18500,2,1,25000,50000,0,4906.177930273105
+e18500,2,1,50000,75000,0,8928.666101610017
+e18500,2,1,75000,100000,0,8963.75338533295
+e18500,2,1,100000,200000,0,17422.439140049133
+e18500,2,1,200000,500000,0,6201.0928840088955
+e18500,2,1,500000,9e99,0,1655.3788738142237
 e26270,0,1,25000,50000,0,6156469.0955616925
 e26270,0,1,50000,75000,0,14034090.695977446
 e26270,0,1,75000,100000,0,18172929.964895956
 e26270,0,1,100000,200000,0,93615898.3292707
 e26270,0,1,200000,500000,0,223182339.6909409
 e26270,0,1,500000,9e99,0,658270024.2177019
-e18400,0,1,10000,25000,0,1730298.162816642
-e18400,0,1,25000,50000,0,12059344.362978864
-e18400,0,1,50000,75000,0,26226702.33681827
-e18400,0,1,75000,100000,0,35963539.38449767
-e18400,0,1,100000,200000,0,115795984.63439415
-e18400,0,1,200000,500000,0,105493210.44703956
-e18400,0,1,500000,9e99,0,225682871.0409268
-e18500,0,1,10000,25000,0,8307917.037332345
-e18500,0,1,25000,50000,0,21631077.550650336
-e18500,0,1,50000,75000,0,39285321.10971896
-e18500,0,1,75000,100000,0,42273138.739311695
-e18500,0,1,100000,200000,0,101295518.30666286
-e18500,0,1,200000,500000,0,58314192.130540885
-e18500,0,1,500000,9e99,0,33548786.640014287
-e02400,0,1,1,10000,0,171298809.01726255
-e02400,0,1,10000,25000,0,317693726.0086798
-e02400,0,1,25000,50000,0,413944478.5434575
-e02400,0,1,50000,75000,0,365059932.9543817
-e02400,0,1,75000,100000,0,314140783.0287367
-e02400,0,1,100000,200000,0,521367504.32092035
-e02400,0,1,200000,500000,0,127240255.18700288
-e02400,0,1,500000,9e99,0,23150322.73235176

tmd/areas/targets/prepare/prepare_cds/.gitignore

@@ -5,9 +5,10 @@
 
 # folders to ignore (anywhere in project since not preceded by root /)
 .Rproj.user/
+_cdprep/
 _docs/
-_targetprep/
 _freeze/
+archive/
 libs/
 # Local Netlify folder
 .netlify

tmd/areas/targets/prepare/prepare_cds/.renvignore

@@ -0,0 +1,7 @@
+
+# folders to avoid scanning by renv
+_cdprep/
+archive/
+data/
+images/
+

tmd/areas/targets/prepare/prepare_cds/R/constants.R

@@ -2,10 +2,9 @@
 CDZIPURL <- "https://www.irs.gov/pub/irs-soi/congressional2021.zip"
 CDDOCURL <- "https://www.irs.gov/pub/irs-soi/21incddocguide.docx"
 
-CDDIR <- here::here("cds")
-CDRAW <- fs::path(CDDIR, "raw_data")
-CDINTERMEDIATE <- fs::path(CDDIR, "intermediate")
-CDFINAL <- fs::path(CDDIR, "final")
+CDDATA <- here::here("data")
+CDRAW <- fs::path(CDDATA, "data_raw")
+CDINTERMEDIATE <- fs::path(CDDATA, "intermediate")
 
 CDDOCEXTRACT <- "cd_documentation_extracted_from_21incddocguide.docx.xlsx"
 

tmd/areas/targets/prepare/prepare_cds/_quarto.yml

@@ -1,6 +1,6 @@
 project:
   type: book
-  output-dir: _targetprep
+  output-dir: _cdprep
 
 # https://prerelease.quarto.org/  # quarto documentation at this link
 
@@ -12,10 +12,10 @@ project:
 
 # publishing with netlify cli:
 #  open terminal in prepare
-#  quarto render && netlify deploy --prod --dir=_targetprep
+#  quarto render && netlify deploy --prod --dir=_cdprep
 
 #  quarto render # inspect to be sure it is as desired
-#  netlify deploy --prod --dir=_targetprep
+#  netlify deploy --prod --dir=_cdprep
 
 # or step by step
 #  netlify deploy # to test it, give _examine as publish directory
@@ -40,20 +40,22 @@ book:
         - usage.qmd
     - part: "IRS SOI Congressional District data"
       chapters:
+        # - cd_agilabels.qmd
         - cd_download_and_clean_census_population_data.qmd
         - cd_download_soi_data.qmd
         - cd_construct_soi_variable_documentation.qmd
         - cd_construct_long_soi_data_file.qmd
         - cd_create_basefile_for_117Congress_cd_target_files.qmd
+        - cd_create_additional_cd_targets.qmd
+        - cd_combine_base_and_additional_targets.qmd
         - cd_create_cd_117_118_crosswalk_and_cdbasefile_118.qmd
-        - cd_create_basefile_multiple_sessions.qmd
-        - cd_create_variable_mapping.qmd
-        - cd_compare_us_totals_tmd_vs_irs_published.qmd
-        - cd_enhance_basefile_with_special_targets.qmd
+        - cd_create_final_enhanced_multisession_basefile.qmd
+        # - cd_create_variable_mapping.qmd
+        # - cd_compare_us_totals_tmd_vs_irs_published.qmd
   appendices:
-    - cd_issues_and_TODOs.qmd  
+    # - cd_issues_and_TODOs.qmd  
     - cd_IRS_documentation.qmd
-        
+
 format:
   html:
     theme: cosmo