Merge pull request #103 from AgenceBio/fix/export-asp-numerobio

AgenceBio · Oct 23, 2023 · 2a91649 · 2a91649
2 parents 28a0c12 + d88445f
commit 2a91649
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 44 deletions.
diff --git a/README.md b/README.md
@@ -151,16 +151,21 @@ tippecanoe -Z10 -z14 --extend-zooms-if-still-dropping --no-tile-compression --si
 
 ## Exporter pour l'ASP
 
+La commande produit un fichier GeoJSON. Elle accepte deux arguments, optionnels :
+
+- `--only-c1` : filtre les parcelles à celles en C1, certifiées jusqu'au 19/09 de l'année en cours, et auditées avant le 15/05 de l'année en cours ;
+- `--numerobio-path` : chemin du fichier CSV où consigner la liste des numéro bio des parcelles exportées.
+
 ### La couche au 15 mai (tout)
 
 ```bash
-docker exec cartobio-api-production node bin/export-asp.js | jq '.[0]' | ogr2ogr cartobio-asp.gpkg /vsistdin/
+docker exec cartobio-api-production node bin/export-asp.js --numerobio-path cartobio-asp.csv | ogr2ogr cartobio-asp.gpkg /vsistdin/
 ```
 
 ### La couche au 12 octobre (C1 uniquement)
 
 ```bash
-docker exec cartobio-api-production node bin/export-asp.js --only-c1 | jq '.[0]' | ogr2ogr cartobio-asp-c1.gpkg /vsistdin/
+docker exec cartobio-api-production node bin/export-asp.js --only-c1 --numerobio-path cartobio-asp-c1.csv | ogr2ogr cartobio-asp-c1.gpkg /vsistdin/
 ```
 
 [cartobio-front]: https://github.com/agencebio/cartobio-front

diff --git a/bin/export-asp.js b/bin/export-asp.js
@@ -3,67 +3,78 @@
 const db = require('../lib/db.js')
 const QueryStream = require('pg-query-stream')
 const JSONStream = require('jsonstream-next')
+const { feature } = require('@turf/helpers')
 const { pipeline } = require('node:stream/promises')
 const minimist = require('minimist')
+const { createWriteStream } = require('fs')
+const { resolve } = require('path')
 
 const cliOptions = {
   alias: {
+    numerobioPath: ['numerobio-path'],
     onlyC1: ['only-c1']
   },
   boolean: ['only-c1'],
-  string: ['campagne'],
+  string: ['campagne', 'numerobio-path'],
   default: {
-    campagne: new Date().getFullYear()
+    campagne: new Date().getFullYear(),
+    numerobioPath: null
   }
 }
 
-const { campagne, onlyC1 } = minimist(process.argv.slice(2), cliOptions)
+const { campagne, numerobioPath, onlyC1 } = minimist(process.argv.slice(2), cliOptions)
 
 // nouvelles en C1 avant le 15/05/2023
 const maxCampagneDate = new Date(`${campagne}-05-15T23:59:59Z`).toISOString()
 // controlées et certifiées avant le 20/09/2023
-const maxAuditDate = onlyC1 ? new Date(`${campagne}-09-20T00:00:00Z`).toISOString() : maxCampagneDate
+const maxAuditDate = onlyC1 ? new Date(`${campagne}-09-20T23:59:59Z`).toISOString() : maxCampagneDate
 const conversionLevels = onlyC1 ? ['C1'] : ['C1', 'C2', 'C3', 'AB']
+// output numerobio list
+const numerobioResolvedPath = numerobioPath ? resolve(process.cwd(), numerobioPath) : null
+const numerobioStream = numerobioResolvedPath ? createWriteStream(numerobioResolvedPath) : null
+const numerobioSet = new Set()
+
+if (numerobioStream) {
+  numerobioStream.write('numeroBio\n')
+}
 
 const query = new QueryStream(/* sql */`-- pg
--- https://postgis.net/docs/ST_AsGeoJSON.html
-SELECT 'FeatureCollection' as type, json_agg(ST_AsGeoJSON(properties)::json) features
+SELECT
+  numerobio,
+  feature->'properties'->'conversion_niveau' AS conversion_niveau,
+  feature->'properties'->'engagement_date' AS engagement_date,
+  -- https://gis.stackexchange.com/a/297289
+  -- ST_Area(ST_Transform(ST_GeomFromGeoJSON(feature->'geometry'), 4326)::geography) / 10000 as surface_ha,
+  -- ST_Area(ST_Transform(ST_GeomFromGeoJSON(feature->'geometry'), 4326)::geography) as surface_m2,
+  -- https://www.postgresql.org/docs/14/functions-datetime.html
+  certification_date AS certification_date_debut,
+  certification_date + interval '18 months' AS certification_date_fin,
+  --
+  -- on ne gère pas encore les déclassements
+  CASE WHEN feature->'properties'->'declassement' IS NULL THEN '{}'::jsonb else feature->'properties'->'declassement' END AS declassement,
+  -- on ne gère pas encore cette distinction
+  -- false as maraichage_diversifie,
+  feature->'geometry' AS geometry
 FROM (
   SELECT
-    feature->'properties'->'conversion_niveau' AS conversion_niveau,
-    feature->'properties'->'engagement_date' AS engagement_date,
-    -- https://gis.stackexchange.com/a/297289
-    -- ST_Area(ST_Transform(ST_GeomFromGeoJSON(feature->'geometry'), 4326)::geography) / 10000 as surface_ha,
-    -- ST_Area(ST_Transform(ST_GeomFromGeoJSON(feature->'geometry'), 4326)::geography) as surface_m2,
-    -- https://www.postgresql.org/docs/14/functions-datetime.html
-    certification_date AS certification_date_debut,
-    certification_date + interval '18 months' AS certification_date_fin,
-    --
-    -- on ne gère pas encore les déclassements
-    CASE WHEN feature->'properties'->'declassement' IS NULL THEN '{}'::jsonb else feature->'properties'->'declassement' END AS declassement,
-    -- on ne gère pas encore cette distinction
-    -- false as maraichage_diversifie,
-    ST_Transform(ST_GeomFromGeoJSON(feature->'geometry'), 4326) AS geom
-  FROM (
-    SELECT
-      jsonb_array_elements("cartobio_operators"."parcelles" -> 'features') AS feature,
-      jsonb_path_query_first("cartobio_operators"."audit_history", '$[*] ? (@.state == "CERTIFIED").date')::text::date AS certification_date,
-      "cartobio_operators"."numerobio",
-      "cartobio_operators"."created_at"
-    FROM "cartobio_operators"
-    WHERE certification_state = 'CERTIFIED'
-      -- la date de certification est AVANT le 15 mai 20xx (inclus)
-      AND audit_history @? '$[*] ? (@.state == "CERTIFIED" && @.date <= "${maxAuditDate}")'
-  ) features
-  WHERE feature->'properties'->>'conversion_niveau' = ANY ($1)
-    AND (
-      -- avec une date d'engagement inférieure au 15 mai 20xx (inclus)
-      -- ou une date pas renseignée
-      feature->'properties'->>'engagement_date' <= $2
-      OR feature->'properties'->>'engagement_date' = ''
-      OR jsonb_path_exists(feature, '$.properties.engagement_date') = FALSE
-    )
-) properties;`, [conversionLevels, maxCampagneDate], { singleRowMode: true, rowMode: 'one' })
+    jsonb_array_elements("cartobio_operators"."parcelles" -> 'features') AS feature,
+    jsonb_path_query_first("cartobio_operators"."audit_history", '$[*] ? (@.state == "CERTIFIED").date')::text::date AS certification_date,
+    "cartobio_operators"."numerobio",
+    "cartobio_operators"."created_at"
+  FROM "cartobio_operators"
+  WHERE certification_state = 'CERTIFIED'
+    -- la date de certification est AVANT le 15 mai 20xx (inclus)
+    AND audit_history @? '$[*] ? (@.state == "CERTIFIED" && @.date <= "${maxAuditDate}")'
+) features
+WHERE
+  feature->'properties'->>'conversion_niveau' = ANY ($1)
+  AND (
+    -- avec une date d'engagement inférieure au 15 mai 20xx (inclus)
+    -- ou une date pas renseignée
+    feature->'properties'->>'engagement_date' <= $2
+    OR feature->'properties'->>'engagement_date' = ''
+    OR jsonb_path_exists(feature, '$.properties.engagement_date') = FALSE
+  );`, [conversionLevels, maxCampagneDate])
 
 ;(async function main () {
   const client = await db.connect()
@@ -79,7 +90,20 @@ FROM (
 
   await pipeline([
     stream,
-    JSONStream.stringify(),
+    async function * (cursor) {
+      for await (const record of cursor) {
+        const { numerobio, geometry, ...properties } = record
+        const f = feature(geometry, properties)
+
+        if (numerobioStream && !numerobioSet.has(numerobio)) {
+          numerobioSet.add(numerobio)
+          numerobioStream.write(`${numerobio}\n`)
+        }
+
+        yield f
+      }
+    },
+    JSONStream.stringify('{ "type": "FeatureCollection", "features": [\n', '\n,\n', '\n]}\n'),
     process.stdout
   ])
 })()
diff --git a/package.json b/package.json
@@ -68,7 +68,8 @@
   "scripts": {
     "build:communes-centroids": "node bin/build-communes-centroids.js",
     "build:types": "tsc lib/outputs/*.js --declaration --allowJs --emitDeclarationOnly --outDir docs/types",
-    "export-asp": "node bin/export-asp.js | jq '.[0]' | ogr2ogr cartobio-asp.gpkg /vsistdin/",
+    "export-asp": "node bin/export-asp.js --numerobio-path cartobio-asp.csv | ogr2ogr cartobio-asp.gpkg /vsistdin/",
+    "export-asp-c1": "node bin/export-asp.js --only-c1 --numerobio-path cartobio-asp-c1.csv | ogr2ogr cartobio-asp-c1.gpkg /vsistdin/",
     "lint": "eslint --ignore-pattern 'migrations/*' .",
     "prestart": "./node_modules/.bin/db-migrate up",
     "test": "NODE_ENV=test jest --detectOpenHandles --collectCoverage --coverageReporters html --coverageReporters text-summary --coverageReporters lcovonly",