From 5b7d0dd261b8265553ce658b52ebd1c19d8e68eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Tue, 17 Dec 2024 10:15:16 +0100 Subject: [PATCH 1/6] add cron monitor with sentry --- .../siaes/management/commands/update_api_entreprise_fields.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lemarche/siaes/management/commands/update_api_entreprise_fields.py b/lemarche/siaes/management/commands/update_api_entreprise_fields.py index 80dd1b8d5..8aa36376a 100644 --- a/lemarche/siaes/management/commands/update_api_entreprise_fields.py +++ b/lemarche/siaes/management/commands/update_api_entreprise_fields.py @@ -1,6 +1,7 @@ import time from django.db.models import Q +from sentry_sdk.crons import monitor from lemarche.siaes.models import Siae from lemarche.utils.apis import api_slack @@ -33,6 +34,7 @@ def add_arguments(self, parser): parser.add_argument("--siret", type=str, default=None, help="Lancer sur un Siret spécifique") parser.add_argument("--limit", type=int, default=None, help="Limiter le nombre de structures à processer") + @monitor(monitor_slug="update-api-entreprise-fields") def handle(self, *args, **options): self.stdout_info("-" * 80) self.stdout_info("Populating API Entreprise fields...") From 7887eb66eea4a35cf9e6c0418e5247269b74a1ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Tue, 17 Dec 2024 20:00:27 +0100 Subject: [PATCH 2/6] update settings --- config/settings/base.py | 2 +- env.default.sh | 4 ++++ env.docker_default.local | 12 +++++------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 792dac176..159430bac 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -507,7 +507,7 @@ # API Entreprise. # https://dashboard.entreprise.api.gouv.fr/login (login is done through auth.api.gouv.fr) # https://doc.entreprise.api.gouv.fr/ -API_ENTREPRISE_BASE_URL = "https://entreprise.api.gouv.fr/v2" +API_ENTREPRISE_BASE_URL = env.str("API_ENTREPRISE_BASE_URL", "https://entreprise.api.gouv.fr/v3") API_ENTREPRISE_CONTEXT = "emplois.inclusion.beta.gouv.fr" API_ENTREPRISE_RECIPIENT = env.str("API_ENTREPRISE_RECIPIENT", "") API_ENTREPRISE_TOKEN = env.str("API_ENTREPRISE_TOKEN", "") diff --git a/env.default.sh b/env.default.sh index 4d6c1cf1b..7caf4e66d 100644 --- a/env.default.sh +++ b/env.default.sh @@ -29,3 +29,7 @@ export OPENAI_API_BASE="" export OPENAI_API_KEY="" export OPENAI_MODEL="" +# API Entreprise / see https://entreprise.api.gouv.fr/developpeurs#kit-de-mise-en-production +export API_ENTREPRISE_RECIPIENT="" +export API_ENTREPRISE_BASE_URL="https://staging.entreprise.api.gouv.fr/v3" +export API_ENTREPRISE_TOKEN="" diff --git a/env.docker_default.local b/env.docker_default.local index a1d426913..22115c072 100755 --- a/env.docker_default.local +++ b/env.docker_default.local @@ -52,12 +52,10 @@ OPENAI_API_BASE= OPENAI_API_BASE= OPENAI_MODEL= -# ELASTICSEARCH -ELASTICSEARCH_HOST= -ELASTICSEARCH_USERNAME= -ELASTICSEARCH_PASSWORD= -ELASTICSEARCH_INDEX_SIAES= -ELASTICSEARCH_MIN_SCORE=0.9 - # DATACUBE API DATACUBE_API_TOKEN= + +# API Entreprise / see https://entreprise.api.gouv.fr/developpeurs#kit-de-mise-en-production +API_ENTREPRISE_RECIPIENT= +API_ENTREPRISE_BASE_URL="https://staging.entreprise.api.gouv.fr/v3" +API_ENTREPRISE_TOKEN= From 7d77737671b273ebdca22b978a57aaef7038bedc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Tue, 17 Dec 2024 20:03:33 +0100 Subject: [PATCH 3/6] update call for api entreprise v3 --- lemarche/utils/apis/api_entreprise.py | 211 ++++++++++++++------------ 1 file changed, 111 insertions(+), 100 deletions(-) diff --git a/lemarche/utils/apis/api_entreprise.py b/lemarche/utils/apis/api_entreprise.py index e0e2330bf..d00ab6393 100644 --- a/lemarche/utils/apis/api_entreprise.py +++ b/lemarche/utils/apis/api_entreprise.py @@ -1,6 +1,7 @@ # https://github.com/betagouv/itou/blob/master/itou/utils/apis/api_entreprise.py import logging +from dataclasses import dataclass from datetime import date, datetime import requests @@ -13,21 +14,40 @@ logger = logging.getLogger(__name__) - API_ENTREPRISE_REASON = "Mise à jour données Marché de la plateforme de l'Inclusion" -TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S" # "2016-12-31T00:00:00+01:00" # timezone not managed +DATE_FORMAT = "%Y-%m-%d" + + +@dataclass +class Entreprise: + forme_juridique: str + forme_juridique_code: str + + +@dataclass +class Etablissement: + naf: str + is_closed: bool + is_head_office: bool + employees: str + employees_date_reference: str + date_constitution: date + + +@dataclass +class Exercice: + chiffre_affaires: str + date_fin_exercice: date def entreprise_get_or_error(siren, reason="Inscription au marché de l'inclusion"): """ - Obtain company data from entreprises.api.gouv.fr - documentation: https://doc.entreprise.api.gouv.fr/?json#entreprise-v2 + Obtain company data from entreprise.api.gouv.fr + documentation: https://entreprise.api.gouv.fr/developpeurs/openapi#tag/Informations-generales/paths/~1v3~1insee~1sirene~1unites_legales~1%7Bsiren%7D/get # noqa Format info: - - "date_mise_a_jour": 1449183600 + - "date_derniere_mise_a_jour": 1449183600 """ - data = None - entreprise = None error = None query_string = urlencode( @@ -37,43 +57,44 @@ def entreprise_get_or_error(siren, reason="Inscription au marché de l'inclusion "object": reason, } ) - - url = f"{settings.API_ENTREPRISE_BASE_URL}/entreprises/{siren}?{query_string}" + url = f"{settings.API_ENTREPRISE_BASE_URL}/insee/sirene/unites_legales/{siren}?{query_string}" headers = {"Authorization": f"Bearer {settings.API_ENTREPRISE_TOKEN}"} try: r = requests.get(url, headers=headers) r.raise_for_status() - data = r.json() + response = r.json() except requests.exceptions.HTTPError as e: status_code = e.response.status_code - if status_code == 422: - error = f"SIREN « {siren} » non reconnu." - elif status_code == 404: - error = f"SIREN « {siren} » 404 ?" - else: - # logger.error("Error while fetching `%s`: %s", url, e) - error = "Problème de connexion à la base Sirene. Essayez ultérieurement." + match status_code: + case 422: + error = f"SIREN « {siren} » non reconnu." + case 404: + error = f"SIREN « {siren} » 404 ?" + case _: + logger.error("Error while fetching `%s`: %s", url, e) + error = "Problème de connexion à la base Sirene. Essayez ultérieurement." return None, error except requests.ReadTimeout as e: # noqa - # logger.error("Error while fetching `%s`: %s", url, e) + logger.error("Error while fetching `%s`: %s", url, e) error = "The read operation timed out" return None, error - if data and data.get("errors"): - error = data["errors"][0] + if response and response.get("errors"): + error = response["errors"][0] return None, error - if not data.get("entreprise") or not data["entreprise"].get("forme_juridique"): - # logger.error("Invalid format of response from API Entreprise") + data = response["data"] + if not data.get("forme_juridique"): + logger.error(f"Invalid format of response from API Entreprise - Entreprise (siren={siren}): {response}") error = "Le format de la réponse API Entreprise est non valide." return None, error - entreprise = { - "forme_juridique": data["entreprise"]["forme_juridique"], - "forme_juridique_code": data["entreprise"]["forme_juridique_code"], - } + entreprise = Entreprise( + forme_juridique=data["forme_juridique"]["libelle"], + forme_juridique_code=data["forme_juridique"]["code"], + ) return entreprise, None @@ -85,12 +106,11 @@ def siae_update_entreprise(siae): return 0, error update_data = dict() - if entreprise: - if entreprise["forme_juridique"]: - update_data["api_entreprise_forme_juridique"] = entreprise["forme_juridique"] - if entreprise["forme_juridique_code"]: - update_data["api_entreprise_forme_juridique_code"] = entreprise["forme_juridique_code"] + if entreprise.forme_juridique: + update_data["api_entreprise_forme_juridique"] = entreprise.forme_juridique + if entreprise.forme_juridique_code: + update_data["api_entreprise_forme_juridique_code"] = entreprise.forme_juridique_code update_data["api_entreprise_entreprise_last_sync_date"] = timezone.now() Siae.objects.filter(id=siae.id).update(**update_data) @@ -101,17 +121,15 @@ def siae_update_entreprise(siae): def etablissement_get_or_error(siret, reason="Inscription au marché de l'inclusion"): """ - Obtain company data from entreprises.api.gouv.fr - documentation: https://doc.entreprise.api.gouv.fr/?json#etablissements-v2 + Obtain company data from entreprise.api.gouv.fr + documentation: https://entreprise.api.gouv.fr/developpeurs/openapi#tag/Informations-generales/paths/~1v3~1insee~1sirene~1etablissements~1%7Bsiret%7D/get # noqa Format info: - "date_mise_a_jour": 1449183600 - "date_reference": "2014" - - "date_creation_etablissement": 1108594800 + - "date_creation": 1108594800 - "date_fermeture": 1315173600 """ - data = None - etablissement = None error = None query_string = urlencode( @@ -121,55 +139,51 @@ def etablissement_get_or_error(siret, reason="Inscription au marché de l'inclus "object": reason, } ) - - url = f"{settings.API_ENTREPRISE_BASE_URL}/etablissements/{siret}?{query_string}" + url = f"{settings.API_ENTREPRISE_BASE_URL}/insee/sirene/etablissements/{siret}?{query_string}" headers = {"Authorization": f"Bearer {settings.API_ENTREPRISE_TOKEN}"} - try: r = requests.get(url, headers=headers) r.raise_for_status() - data = r.json() + response = r.json() except requests.exceptions.HTTPError as e: status_code = e.response.status_code - - if status_code == 422: - error = f"SIRET « {siret} » non reconnu." - elif status_code == 404: - error = f"SIRET « {siret} » 404 ?" - else: - # logger.error("Error while fetching `%s`: %s", url, e) - error = "Problème de connexion à la base Sirene. Essayez ultérieurement." + match status_code: + case 422: + error = f"SIRET « {siret} » non reconnu." # TODO: check v3 error code + case 404: + error = f"SIRET « {siret} » 404 ?" + case _: + # logger.error("Error while fetching `%s`: %s", url, e) + error = f"Problème de connexion à la base Sirene. Essayez ultérieurement. ({status_code})" return None, error except requests.ReadTimeout as e: # noqa - # logger.error("Error while fetching `%s`: %s", url, e) + logger.error("Error while fetching `%s`: %s", url, e) error = "The read operation timed out" return None, error - if data and data.get("errors"): - error = data["errors"][0] + if response and response.get("errors"): + error = response["errors"][0] return None, error - if not data.get("etablissement") or not data["etablissement"].get("adresse"): - # logger.error("Invalid format of response from API Entreprise") + data = response["data"] + if ( + not data.get("activite_principale") + or not data.get("etat_administratif") + or not data.get("tranche_effectif_salarie") + or not data.get("date_creation") + ): + logger.error(f"Invalid format of response from API Entreprise - Etablissement (siret={siret}): {response}") error = "Le format de la réponse API Entreprise est non valide." return None, error - # address = data["etablissement"]["adresse"] - etablissement = { - # name=address["l1"], - # # FIXME To check (l4 => line_1) - # address_line_1=address["l4"], - # address_line_2=address["l3"], - # post_code=address["code_postal"], - # city=address["localite"], - # department=department_from_postcode(address["code_postal"]), - "naf": data["etablissement"]["naf"], - "is_closed": data["etablissement"]["etat_administratif"]["value"] == "F", - "is_head_office": data["etablissement"].get("siege_social", False), - "employees": data["etablissement"]["tranche_effectif_salarie_etablissement"]["intitule"], - "employees_date_reference": data["etablissement"]["tranche_effectif_salarie_etablissement"]["date_reference"], - "date_constitution": date.fromtimestamp(data["etablissement"]["date_creation_etablissement"]), - } + etablissement = Etablissement( + naf=data["activite_principale"]["code"], + is_closed=data["etat_administratif"] == "F", + is_head_office=data.get("siege_social", False), + employees=data["tranche_effectif_salarie"]["intitule"], + employees_date_reference=data["tranche_effectif_salarie"]["date_reference"], + date_constitution=date.fromtimestamp(data["date_creation"]), + ) return etablissement, None @@ -180,40 +194,36 @@ def siae_update_etablissement(siae): return 0, error update_data = dict() - if etablissement: - # update_data"nature"] = siae_constants.NATURE_HEAD_OFFICE if etablissement["is_head_office"] else siae_constants.NATURE_ANTENNA # noqa - # update_data"is_active"] = False if not etablissement["is_closed"] else True - if etablissement["employees"]: + if etablissement.employees: update_data["api_entreprise_employees"] = ( - etablissement["employees"] - if (etablissement["employees"] != "Unités non employeuses") + etablissement.employees + if (etablissement.employees != "Unités non employeuses") else "Non renseigné" ) - if etablissement["employees_date_reference"]: - update_data["api_entreprise_employees_year_reference"] = etablissement["employees_date_reference"] - if etablissement["date_constitution"]: - update_data["api_entreprise_date_constitution"] = etablissement["date_constitution"] + if etablissement.employees_date_reference: + update_data["api_entreprise_employees_year_reference"] = etablissement.employees_date_reference + if etablissement.date_constitution: + update_data["api_entreprise_date_constitution"] = etablissement.date_constitution update_data["api_entreprise_etablissement_last_sync_date"] = timezone.now() Siae.objects.filter(id=siae.id).update(**update_data) return 1, etablissement + return 0, f"SIAE {siae.id} without SIRET" def exercice_get_or_error(siret, reason="Inscription au marché de l'inclusion"): """ Obtain company data from entreprises.api.gouv.fr - documentation: https://entreprise.api.gouv.fr/catalogue/#a-exercices + documentation: https://entreprise.api.gouv.fr/developpeurs/openapi#tag/Informations-financieres/paths/~1v3~1dgfip~1etablissements~1%7Bsiret%7D~1chiffres_affaires/get # noqa Format info: - - "date_fin_exercice": "2016-12-31T00:00:00+01:00" + - "date_fin_exercice": "2024-12-17" Often returns errors: 404, 422, 502 """ - data = None - exercice = None error = None query_string = urlencode( @@ -223,45 +233,46 @@ def exercice_get_or_error(siret, reason="Inscription au marché de l'inclusion") "object": reason, } ) - - url = f"{settings.API_ENTREPRISE_BASE_URL}/exercices/{siret}?{query_string}" + url = f"{settings.API_ENTREPRISE_BASE_URL}/dgfip/etablissements/{siret}/chiffres_affaires?{query_string}" headers = {"Authorization": f"Bearer {settings.API_ENTREPRISE_TOKEN}"} try: r = requests.get(url, headers=headers) r.raise_for_status() - data = r.json() + response = r.json() except requests.exceptions.HTTPError as e: status_code = e.response.status_code if status_code == 422: error = f"SIRET {siret} non reconnu." else: - # logger.error("Error while fetching `%s`: %s", url, e) - error = "Problème de connexion à la base Sirene. Essayez ultérieurement." + logger.error("Error while fetching `%s`: %s", url, e) + error = f"Problème de connexion à la base Sirene. Essayez ultérieurement. ({status_code})" return None, error except requests.ReadTimeout as e: # noqa - # logger.error("Error while fetching `%s`: %s", url, e) + logger.error("Error while fetching `%s`: %s", url, e) error = "The read operation timed out" return None, error - if data and data.get("errors"): - error = data["errors"][0] + if response and response.get("errors"): + error = response["errors"][0] return None, error - if not data.get("exercices") or not len(data["exercices"]): - # logger.error("Invalid format of response from API Entreprise") + if not response.get("data") or not len(response["data"]): + logger.error(f"Invalid format of response from API Entreprise - Exercice (siret={siret}): {response}") error = "Le format de la réponse API Entreprise est non valide." return None, error - exercice = data["exercices"][0] - + exercice = Exercice( + chiffre_affaires=response["data"][0]["data"]["chiffre_affaires"], + date_fin_exercice=response["data"][0]["data"]["date_fin_exercice"], + ) return exercice, None def siae_update_exercice(siae): if siae.siret: - exercice, error = exercice_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) # noqa + exercice, error = exercice_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) if error: return 0, error @@ -269,11 +280,11 @@ def siae_update_exercice(siae): if exercice: update_data = dict() - if exercice["ca"]: - update_data["api_entreprise_ca"] = exercice["ca"] - if exercice["date_fin_exercice"]: + if exercice.chiffre_affaires: + update_data["api_entreprise_ca"] = exercice.chiffre_affaires + if exercice.date_fin_exercice: update_data["api_entreprise_ca_date_fin_exercice"] = datetime.strptime( - exercice["date_fin_exercice"][:-6], TIMESTAMP_FORMAT + exercice.date_fin_exercice, DATE_FORMAT ).date() update_data["api_entreprise_exercice_last_sync_date"] = timezone.now() From 7732a25dc08c4e64cc56767ccef8978ce9ff7410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Tue, 17 Dec 2024 20:06:12 +0100 Subject: [PATCH 4/6] add tests for api entreprise helper --- lemarche/utils/tests_apis.py | 256 +++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 lemarche/utils/tests_apis.py diff --git a/lemarche/utils/tests_apis.py b/lemarche/utils/tests_apis.py new file mode 100644 index 000000000..ec8e7bcaf --- /dev/null +++ b/lemarche/utils/tests_apis.py @@ -0,0 +1,256 @@ +import datetime +import json +from unittest.mock import patch + +from django.test import TestCase +from django.utils import timezone + +from lemarche.siaes.factories import SiaeFactory +from lemarche.utils.apis.api_entreprise import siae_update_entreprise, siae_update_etablissement, siae_update_exercice + + +MOCK_ENTREPRISE_API_DATA = """ +{ + "data": { + "siren": "130025265", + "rna": "W751004076", + "siret_siege_social": "13002526500013", + "type": "personne_morale", + "personne_morale_attributs": { + "raison_sociale": "DIRECTION INTERMINISTERIELLE DU NUMERIQUE", + "sigle": "DINUM" + }, + "personne_physique_attributs": { + "pseudonyme": "DJ Falcon", + "prenom_usuel": "Jean", + "prenom_1": "Jean", + "prenom_2": "Jacques", + "prenom_3": "Pierre", + "prenom_4": "Paul", + "nom_usage": "Dupont", + "nom_naissance": "Martin", + "sexe": "M" + }, + "categorie_entreprise": "GE", + "status_diffusion": "diffusible", + "diffusable_commercialement": true, + "forme_juridique": { + "code": "7120", + "libelle": "Service central d'un ministère" + }, + "activite_principale": { + "code": "8411Z", + "libelle": "Administration publique générale", + "nomenclature": "NAFRev2" + }, + "tranche_effectif_salarie": { + "code": "51", + "intitule": "2 000 à 4 999 salariés", + "date_reference": "2016", + "de": 2000, + "a": 4999 + }, + "etat_administratif": "A", + "economie_sociale_et_solidaire": true, + "date_creation": 1634103818, + "date_cessation": 1634133818 + }, + "links": { + "siege_social": "https://entreprises.api.gouv.fr/api/v3/insee/etablissements/30613890001294", + "siege_social_adresse": "https://entreprises.api.gouv.fr/api/v3/insee/etablissements/30613890001294/adresse" + }, + "meta": { + "date_derniere_mise_a_jour": 1618396818, + "redirect_from_siren": "306138900" + } +} +""" + + +class TestSiaeUpdateEntreprise(TestCase): + def setUp(self): + self.siae = SiaeFactory(siret="13002526500013") + + @patch("requests.get") + def test_siae_update_entreprise(self, mock_api): + mock_response = mock_api.return_value + mock_response.json.return_value = json.loads(MOCK_ENTREPRISE_API_DATA) + mock_response.status_code = 200 + + result, entreprise = siae_update_entreprise(self.siae) + + self.siae.refresh_from_db() + + # Assert the result + self.assertEqual(result, 1) + self.assertIsNotNone(entreprise) + + # Assert the updates + self.assertEqual(self.siae.api_entreprise_forme_juridique, "Service central d'un ministère") + self.assertEqual(self.siae.api_entreprise_forme_juridique_code, "7120") + self.assertLess((timezone.now() - self.siae.api_entreprise_entreprise_last_sync_date).total_seconds(), 60) + + +MOCK_ETABLISSEMENT_API_DATA = """ +{ + "data": { + "siret": "30613890001294", + "siege_social": true, + "etat_administratif": "A", + "date_fermeture": 1634133818, + "status_diffusion": "diffusible", + "activite_principale": { + "code": "8411Z", + "libelle": "Administration publique générale", + "nomenclature": "NAFRev2" + }, + "tranche_effectif_salarie": { + "code": "51", + "intitule": "2 000 à 4 999 salariés", + "date_reference": "2016", + "de": 2000, + "a": 4999 + }, + "diffusable_commercialement": true, + "enseigne": "Coiff Land, CoiffureLand", + "unite_legale": { + "siren": "130025265", + "rna": null, + "siret_siege_social": "13002526500013", + "type": "personne_morale", + "status_diffusion": "diffusible", + "personne_morale_attributs": { + "raison_sociale": "DIRECTION INTERMINISTERIELLE DU NUMERIQUE", + "sigle": "DINUM" + }, + "personne_physique_attributs": { + "pseudonyme": "DJ Falcon", + "prenom_usuel": "Jean", + "prenom_1": "Jean", + "prenom_2": "Jacques", + "prenom_3": "Pierre", + "prenom_4": "Paul", + "nom_usage": "Dupont", + "nom_naissance": "Dubois", + "sexe": "M" + }, + "categorie_entreprise": "GE", + "diffusable_commercialement": true, + "forme_juridique": { + "code": "7120", + "libelle": "Service central d'un ministère" + }, + "activite_principale": { + "code": "8411Z", + "libelle": "Administration publique générale", + "nomenclature": "NAFRev2" + }, + "tranche_effectif_salarie": { + "code": "51", + "intitule": "2 000 à 4 999 salariés", + "date_reference": "2016", + "de": 2000, + "a": 4999 + }, + "etat_administratif": "A", + "economie_sociale_et_solidaire": true, + "date_creation": 1634103818 + }, + "adresse": { + "numero_voie": "22", + "indice_repetition_voie": null, + "type_voie": "RUE", + "libelle_voie": "DE LA PAIX", + "complement_adresse": "ZAE SAINT GUENAULT", + "code_commune": "75112", + "code_postal": "75016", + "distribution_speciale": "dummy", + "code_cedex": "75590", + "libelle_cedex": "PARIS CEDEX 12", + "libelle_commune": "PARIS 12", + "libelle_commune_etranger": "dummy", + "code_pays_etranger": "99132", + "libelle_pays_etranger": "ROYAUME-UNI", + "status_diffusion": "diffusible", + "acheminement_postal": { + "l1": "DIRECTION INTERMINISTERIELLE DU NUMERIQUE", + "l2": "JEAN MARIE DURAND", + "l3": "ZAE SAINT GUENAULT", + "l4": "51 BIS RUE DE LA PAIX", + "l5": "CS 72809", + "l6": "75256 PARIX CEDEX 12", + "l7": "FRANCE" + } + }, + "date_creation": 1634103818 + }, + "links": { + "unite_legale": "https://entreprise.api.gouv.fr/api/v3/insee/unites_legales/130025265" + }, + "meta": { + "date_derniere_mise_a_jour": 1618396818, + "redirect_from_siret": "30613890000010" + } +} +""" + + +class TestSiaeUpdateEtablissement(TestCase): + def setUp(self): + self.siae = SiaeFactory(siret="30613890001294") + + @patch("requests.get") + def test_siae_update_etablissement(self, mock_api): + mock_response = mock_api.return_value + mock_response.json.return_value = json.loads(MOCK_ETABLISSEMENT_API_DATA) + mock_response.status_code = 200 + + result, etablissement = siae_update_etablissement(self.siae) + + # Assert the result + self.assertEqual(result, 1) + self.assertIsNotNone(etablissement) + + # Assert the updates + self.siae.refresh_from_db() + self.assertEqual(self.siae.siret, "30613890001294") + self.assertEqual(self.siae.api_entreprise_employees, "2 000 à 4 999 salariés") + self.assertEqual(self.siae.api_entreprise_employees_year_reference, "2016") + self.assertEqual(self.siae.api_entreprise_date_constitution, datetime.date(2021, 10, 13)) + self.assertLess((timezone.now() - self.siae.api_entreprise_etablissement_last_sync_date).total_seconds(), 60) + + +MOCK_EXERCICES_API_DATA = """ +{ + "data": [ + { + "data": { + "chiffre_affaires": 900001, + "date_fin_exercice": "2015-12-01" + }, + "links": {}, + "meta": {} + } + ], + "meta": {}, + "links": {} +} +""" + + +class TestSiaeUpdateExercice(TestCase): + def setUp(self): + self.siae = SiaeFactory(siret="30613890001294") + + @patch("requests.get") + def test_siae_update_exercice(self, mock_api): + mock_response = mock_api.return_value + mock_response.json.return_value = json.loads(MOCK_EXERCICES_API_DATA) + mock_response.status_code = 200 + + siae_update_exercice(self.siae) + + self.siae.refresh_from_db() + self.assertEqual(self.siae.api_entreprise_ca, 900001) + self.assertEqual(self.siae.api_entreprise_ca_date_fin_exercice, datetime.date(2015, 12, 1)) + self.assertLess((timezone.now() - self.siae.api_entreprise_exercice_last_sync_date).total_seconds(), 60) From 605bea87076aa7c18582380387f5e6b5c795cfde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Thu, 19 Dec 2024 17:12:32 +0100 Subject: [PATCH 5/6] refactor: merge the three updates into one --- .../commands/update_api_entreprise_fields.py | 201 ++++++++---------- lemarche/siaes/tests/test_commands.py | 62 +++++- lemarche/utils/apis/api_entreprise.py | 116 ++-------- .../api_entreprise.py} | 83 +------- 4 files changed, 169 insertions(+), 293 deletions(-) rename lemarche/utils/{tests_apis.py => mocks/api_entreprise.py} (65%) diff --git a/lemarche/siaes/management/commands/update_api_entreprise_fields.py b/lemarche/siaes/management/commands/update_api_entreprise_fields.py index 8aa36376a..9200aa115 100644 --- a/lemarche/siaes/management/commands/update_api_entreprise_fields.py +++ b/lemarche/siaes/management/commands/update_api_entreprise_fields.py @@ -1,11 +1,18 @@ import time +from datetime import datetime from django.db.models import Q +from django.utils import timezone from sentry_sdk.crons import monitor from lemarche.siaes.models import Siae from lemarche.utils.apis import api_slack -from lemarche.utils.apis.api_entreprise import siae_update_entreprise, siae_update_etablissement, siae_update_exercice +from lemarche.utils.apis.api_entreprise import ( + API_ENTREPRISE_REASON, + entreprise_get_or_error, + etablissement_get_or_error, + exercice_get_or_error, +) from lemarche.utils.commands import BaseCommand @@ -54,124 +61,96 @@ def handle(self, *args, **options): if options["limit"]: siae_queryset = siae_queryset[: options["limit"]] - # self.stdout_info(f"Found {siae_queryset.count()} Siae") + results = { + "entreprise": {"success": 0, "error": 0}, + "etablissement": {"success": 0, "error": 0}, + "exercice": {"success": 0, "error": 0}, + } - if options["scope"] in ("all", "entreprise"): - self.update_api_entreprise_entreprise_fields(siae_queryset) - - if options["scope"] in ("all", "etablissement"): - self.update_api_entreprise_etablissement_fields(siae_queryset) - - if options["scope"] in ("all", "exercice"): - self.update_api_entreprise_exercice_fields(siae_queryset) - - # API Entreprise: entreprises - def update_api_entreprise_entreprise_fields(self, siae_queryset): progress = 0 - results = {"success": 0, "error": 0} - siae_queryset_entreprise = siae_queryset.filter(api_entreprise_entreprise_last_sync_date=None) - self.stdout_info("-" * 80) - self.stdout_info(f"Populating 'entreprise' for {siae_queryset_entreprise.count()} Siae...") - - for siae in siae_queryset_entreprise: - try: - progress += 1 - if (progress % 50) == 0: - self.stdout_info(f"{progress}...") - # self.stdout_info("-" * 80) - # self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}") - response, message = siae_update_entreprise(siae) - if response: - results["success"] += 1 - else: - self.stdout_error(str(message)) - results["error"] += 1 - # small delay to avoid going above the API limitation - # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints" - time.sleep(0.5) - except Exception as e: - self.stdout_error(str(e)) - api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: entreprises") + for siae in siae_queryset: + progress += 1 + if (progress % 50) == 0: + self.stdout_info(f"{progress}...") + + if siae.siret: + update_data = dict() + if options["scope"] in ("all", "entreprise") and siae.api_entreprise_entreprise_last_sync_date is None: + entreprise, error = entreprise_get_or_error(siae.siret[:9], reason=API_ENTREPRISE_REASON) + if error: + results["entreprise"]["error"] += 1 + self.stdout_error(str(error)) + else: + results["entreprise"]["success"] += 1 + if entreprise: + if entreprise.forme_juridique: + update_data["api_entreprise_forme_juridique"] = entreprise.forme_juridique + if entreprise.forme_juridique_code: + update_data["api_entreprise_forme_juridique_code"] = entreprise.forme_juridique_code + update_data["api_entreprise_entreprise_last_sync_date"] = timezone.now() + + if ( + options["scope"] in ("all", "etablissement") + and siae.api_entreprise_etablissement_last_sync_date is None + ): + etablissement, error = etablissement_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) + if error: + results["etablissement"]["error"] += 1 + self.stdout_error(str(error)) + else: + results["etablissement"]["success"] += 1 + if etablissement: + if etablissement.employees: + update_data["api_entreprise_employees"] = ( + etablissement.employees + if (etablissement.employees != "Unités non employeuses") + else "Non renseigné" + ) + if etablissement.employees_date_reference: + update_data[ + "api_entreprise_employees_year_reference" + ] = etablissement.employees_date_reference + if etablissement.date_constitution: + update_data["api_entreprise_date_constitution"] = etablissement.date_constitution + + update_data["api_entreprise_etablissement_last_sync_date"] = timezone.now() + + if options["scope"] in ("all", "exercice") and siae.api_entreprise_exercice_last_sync_date is None: + exercice, error = exercice_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) + if error: + results["exercice"]["error"] += 1 + self.stdout_error(str(error)) + else: + results["exercice"]["success"] += 1 + if exercice: + if exercice.chiffre_affaires: + update_data["api_entreprise_ca"] = exercice.chiffre_affaires + if exercice.date_fin_exercice: + update_data["api_entreprise_ca_date_fin_exercice"] = datetime.strptime( + exercice.date_fin_exercice, "%Y-%m-%d" + ).date() + + update_data["api_entreprise_exercice_last_sync_date"] = timezone.now() + + Siae.objects.filter(id=siae.id).update(**update_data) - msg_success = [ - "----- Synchronisation API Entreprise (/entreprises) -----", - f"Done! Processed {siae_queryset_entreprise.count()} siae", - f"success count: {results['success']}/{siae_queryset_entreprise.count()}", - f"error count: {results['error']}/{siae_queryset_entreprise.count()} (voir les logs)", - ] - self.stdout_messages_success(msg_success) - api_slack.send_message_to_channel("\n".join(msg_success)) - - # API Entreprise: etablissements - def update_api_entreprise_etablissement_fields(self, siae_queryset): - progress = 0 - results = {"success": 0, "error": 0} - siae_queryset_etablissement = siae_queryset.filter(api_entreprise_etablissement_last_sync_date=None) - self.stdout_info("-" * 80) - self.stdout_info(f"Populating 'etablissement' for {siae_queryset_etablissement.count()} Siae...") - - for siae in siae_queryset_etablissement: - try: - progress += 1 - if (progress % 50) == 0: - self.stdout_info(f"{progress}...") - # self.stdout_info("-" * 80) - # self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}") - response, message = siae_update_etablissement(siae) - if response: - results["success"] += 1 - else: - self.stdout_error(str(message)) - results["error"] += 1 - # small delay to avoid going above the API limitation - # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints" - time.sleep(0.5) - except Exception as e: - self.stdout_error(str(e)) - api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: etablissements") - - msg_success = [ - "----- Synchronisation API Entreprise (/etablissements) -----", - f"Done! Processed {siae_queryset_etablissement.count()} siae", - f"success count: {results['success']}/{siae_queryset_etablissement.count()}", - f"error count: {results['error']}/{siae_queryset_etablissement.count()} (voir les logs)", - ] - self.stdout_messages_success(msg_success) - api_slack.send_message_to_channel("\n".join(msg_success)) - - # API Entreprise: exercices - def update_api_entreprise_exercice_fields(self, siae_queryset): - progress = 0 - results = {"success": 0, "error": 0} - siae_queryset_exercice = siae_queryset.filter(api_entreprise_exercice_last_sync_date=None) - self.stdout_info("-" * 80) - self.stdout_info(f"Populating 'exercice' for {siae_queryset_exercice.count()} Siae...") - - for siae in siae_queryset_exercice: - try: - progress += 1 - if (progress % 50) == 0: - self.stdout_info(f"{progress}...") - # self.stdout_info("-" * 80) - # self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}") - response, message = siae_update_exercice(siae) - if response: - results["success"] += 1 - else: - self.stdout_error(str(message)) - results["error"] += 1 # small delay to avoid going above the API limitation # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints" time.sleep(0.5) - except Exception as e: - self.stdout_error(str(e)) - api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: exercices") + else: + self.stdout_error(f"SIAE {siae.id} without SIRET") msg_success = [ - "----- Synchronisation API Entreprise (/exercices) -----", - f"Done! Processed {siae_queryset_exercice.count()} siae", - f"success count: {results['success']}/{siae_queryset_exercice.count()}", - f"error count: {results['error']}/{siae_queryset_exercice.count()} (voir les logs)", + "----- Synchronisation API Entreprise -----", + f"Done! Processed {siae_queryset.count()} siae", + "----- Success -----", + f"entreprise: {results['entreprise']['success']}/{siae_queryset.count()}", + f"etablissement: {results['etablissement']['success']}/{siae_queryset.count()}", + f"exercice: {results['exercice']['success']}/{siae_queryset.count()}", + "----- Error ----- (voir les logs)", + f"entreprise: {results['entreprise']['error']}/{siae_queryset.count()}", + f"etablissement: {results['etablissement']['error']}/{siae_queryset.count()}", + f"exercice: {results['exercice']['error']}/{siae_queryset.count()}", ] self.stdout_messages_success(msg_success) api_slack.send_message_to_channel("\n".join(msg_success)) diff --git a/lemarche/siaes/tests/test_commands.py b/lemarche/siaes/tests/test_commands.py index b78cc1541..9f5eb55c3 100644 --- a/lemarche/siaes/tests/test_commands.py +++ b/lemarche/siaes/tests/test_commands.py @@ -1,11 +1,14 @@ -import factory +import datetime +import json import logging import os from unittest.mock import patch +import factory from django.core.management import call_command from django.db.models import signals from django.test import TransactionTestCase +from django.utils import timezone from lemarche.perimeters.factories import PerimeterFactory from lemarche.perimeters.models import Perimeter @@ -14,6 +17,11 @@ from lemarche.siaes.factories import SiaeActivityFactory, SiaeFactory from lemarche.siaes.models import Siae, SiaeActivity from lemarche.users.factories import UserFactory +from lemarche.utils.mocks.api_entreprise import ( + MOCK_ENTREPRISE_API_DATA, + MOCK_ETABLISSEMENT_API_DATA, + MOCK_EXERCICES_API_DATA, +) class SyncWithEmploisInclusionCommandTest(TransactionTestCase): @@ -524,3 +532,55 @@ def test_update_count_fields_with_id(self): siae_not_updated.refresh_from_db() self.assertEqual(siae_not_updated.user_count, 0) self.assertEqual(siae_not_updated.sector_count, 0) + + +class SiaeUpdateApiEntrepriseFieldsCommandTest(TransactionTestCase): + @patch("requests.get") + def test_siae_update_entreprise(self, mock_api): + mock_response = mock_api.return_value + mock_response.json.return_value = json.loads(MOCK_ENTREPRISE_API_DATA) + mock_response.status_code = 200 + + siae = SiaeFactory(siret="13002526500013") + + call_command("update_api_entreprise_fields", scope="entreprise") + + # Assert the updates + siae.refresh_from_db() + self.assertEqual(siae.api_entreprise_forme_juridique, "Service central d'un ministère") + self.assertEqual(siae.api_entreprise_forme_juridique_code, "7120") + self.assertLess((timezone.now() - siae.api_entreprise_entreprise_last_sync_date).total_seconds(), 60) + + @patch("requests.get") + def test_siae_update_etablissement(self, mock_api): + mock_response = mock_api.return_value + mock_response.json.return_value = json.loads(MOCK_ETABLISSEMENT_API_DATA) + mock_response.status_code = 200 + + siae = SiaeFactory(siret="30613890001294") + + call_command("update_api_entreprise_fields", scope="etablissement") + + # Assert the updates + siae.refresh_from_db() + self.assertEqual(siae.siret, "30613890001294") + self.assertEqual(siae.api_entreprise_employees, "2 000 à 4 999 salariés") + self.assertEqual(siae.api_entreprise_employees_year_reference, "2016") + self.assertEqual(siae.api_entreprise_date_constitution, datetime.date(2021, 10, 13)) + self.assertLess((timezone.now() - siae.api_entreprise_etablissement_last_sync_date).total_seconds(), 60) + + @patch("requests.get") + def test_siae_update_exercice(self, mock_api): + mock_response = mock_api.return_value + mock_response.json.return_value = json.loads(MOCK_EXERCICES_API_DATA) + mock_response.status_code = 200 + + siae = SiaeFactory(siret="30613890001294") + + call_command("update_api_entreprise_fields", scope="exercice") + + # Assert the updates + siae.refresh_from_db() + self.assertEqual(siae.api_entreprise_ca, 900001) + self.assertEqual(siae.api_entreprise_ca_date_fin_exercice, datetime.date(2015, 12, 1)) + self.assertLess((timezone.now() - siae.api_entreprise_exercice_last_sync_date).total_seconds(), 60) diff --git a/lemarche/utils/apis/api_entreprise.py b/lemarche/utils/apis/api_entreprise.py index d00ab6393..eca055df0 100644 --- a/lemarche/utils/apis/api_entreprise.py +++ b/lemarche/utils/apis/api_entreprise.py @@ -2,20 +2,16 @@ import logging from dataclasses import dataclass -from datetime import date, datetime +from datetime import date import requests from django.conf import settings -from django.utils import timezone from django.utils.http import urlencode -from lemarche.siaes.models import Siae - logger = logging.getLogger(__name__) API_ENTREPRISE_REASON = "Mise à jour données Marché de la plateforme de l'Inclusion" -DATE_FORMAT = "%Y-%m-%d" @dataclass @@ -40,6 +36,17 @@ class Exercice: date_fin_exercice: date +def get_url_endpoint(endpoint: str, reason: str) -> str: + query_string = urlencode( + { + "recipient": settings.API_ENTREPRISE_RECIPIENT, + "context": settings.API_ENTREPRISE_CONTEXT, + "object": reason, + } + ) + return f"{settings.API_ENTREPRISE_BASE_URL}/{endpoint}?{query_string}" + + def entreprise_get_or_error(siren, reason="Inscription au marché de l'inclusion"): """ Obtain company data from entreprise.api.gouv.fr @@ -50,14 +57,7 @@ def entreprise_get_or_error(siren, reason="Inscription au marché de l'inclusion """ error = None - query_string = urlencode( - { - "recipient": settings.API_ENTREPRISE_RECIPIENT, - "context": settings.API_ENTREPRISE_CONTEXT, - "object": reason, - } - ) - url = f"{settings.API_ENTREPRISE_BASE_URL}/insee/sirene/unites_legales/{siren}?{query_string}" + url = get_url_endpoint(f"insee/sirene/unites_legales/{siren}", reason) headers = {"Authorization": f"Bearer {settings.API_ENTREPRISE_TOKEN}"} try: @@ -98,27 +98,6 @@ def entreprise_get_or_error(siren, reason="Inscription au marché de l'inclusion return entreprise, None -def siae_update_entreprise(siae): - if siae.siret: - siae_siren = siae.siret[:9] - entreprise, error = entreprise_get_or_error(siae_siren, reason=API_ENTREPRISE_REASON) - if error: - return 0, error - - update_data = dict() - if entreprise: - if entreprise.forme_juridique: - update_data["api_entreprise_forme_juridique"] = entreprise.forme_juridique - if entreprise.forme_juridique_code: - update_data["api_entreprise_forme_juridique_code"] = entreprise.forme_juridique_code - - update_data["api_entreprise_entreprise_last_sync_date"] = timezone.now() - Siae.objects.filter(id=siae.id).update(**update_data) - - return 1, entreprise - return 0, f"SIAE {siae.id} without SIREN" - - def etablissement_get_or_error(siret, reason="Inscription au marché de l'inclusion"): """ Obtain company data from entreprise.api.gouv.fr @@ -132,14 +111,7 @@ def etablissement_get_or_error(siret, reason="Inscription au marché de l'inclus """ error = None - query_string = urlencode( - { - "recipient": settings.API_ENTREPRISE_RECIPIENT, - "context": settings.API_ENTREPRISE_CONTEXT, - "object": reason, - } - ) - url = f"{settings.API_ENTREPRISE_BASE_URL}/insee/sirene/etablissements/{siret}?{query_string}" + url = get_url_endpoint(f"insee/sirene/etablissements/{siret}", reason) headers = {"Authorization": f"Bearer {settings.API_ENTREPRISE_TOKEN}"} try: r = requests.get(url, headers=headers) @@ -187,33 +159,6 @@ def etablissement_get_or_error(siret, reason="Inscription au marché de l'inclus return etablissement, None -def siae_update_etablissement(siae): - if siae.siret: - etablissement, error = etablissement_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) - if error: - return 0, error - - update_data = dict() - if etablissement: - if etablissement.employees: - update_data["api_entreprise_employees"] = ( - etablissement.employees - if (etablissement.employees != "Unités non employeuses") - else "Non renseigné" - ) - if etablissement.employees_date_reference: - update_data["api_entreprise_employees_year_reference"] = etablissement.employees_date_reference - if etablissement.date_constitution: - update_data["api_entreprise_date_constitution"] = etablissement.date_constitution - - update_data["api_entreprise_etablissement_last_sync_date"] = timezone.now() - Siae.objects.filter(id=siae.id).update(**update_data) - - return 1, etablissement - - return 0, f"SIAE {siae.id} without SIRET" - - def exercice_get_or_error(siret, reason="Inscription au marché de l'inclusion"): """ Obtain company data from entreprises.api.gouv.fr @@ -226,14 +171,7 @@ def exercice_get_or_error(siret, reason="Inscription au marché de l'inclusion") """ error = None - query_string = urlencode( - { - "recipient": settings.API_ENTREPRISE_RECIPIENT, - "context": settings.API_ENTREPRISE_CONTEXT, - "object": reason, - } - ) - url = f"{settings.API_ENTREPRISE_BASE_URL}/dgfip/etablissements/{siret}/chiffres_affaires?{query_string}" + url = get_url_endpoint(f"dgfip/etablissements/{siret}/chiffres_affaires", reason) headers = {"Authorization": f"Bearer {settings.API_ENTREPRISE_TOKEN}"} try: @@ -268,27 +206,3 @@ def exercice_get_or_error(siret, reason="Inscription au marché de l'inclusion") date_fin_exercice=response["data"][0]["data"]["date_fin_exercice"], ) return exercice, None - - -def siae_update_exercice(siae): - if siae.siret: - exercice, error = exercice_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) - if error: - return 0, error - - update_data = dict() - - if exercice: - update_data = dict() - if exercice.chiffre_affaires: - update_data["api_entreprise_ca"] = exercice.chiffre_affaires - if exercice.date_fin_exercice: - update_data["api_entreprise_ca_date_fin_exercice"] = datetime.strptime( - exercice.date_fin_exercice, DATE_FORMAT - ).date() - - update_data["api_entreprise_exercice_last_sync_date"] = timezone.now() - Siae.objects.filter(id=siae.id).update(**update_data) - - return 1, exercice - return 0, f"SIAE {siae.id} without SIRET" diff --git a/lemarche/utils/tests_apis.py b/lemarche/utils/mocks/api_entreprise.py similarity index 65% rename from lemarche/utils/tests_apis.py rename to lemarche/utils/mocks/api_entreprise.py index ec8e7bcaf..170c110f0 100644 --- a/lemarche/utils/tests_apis.py +++ b/lemarche/utils/mocks/api_entreprise.py @@ -1,14 +1,4 @@ -import datetime -import json -from unittest.mock import patch - -from django.test import TestCase -from django.utils import timezone - -from lemarche.siaes.factories import SiaeFactory -from lemarche.utils.apis.api_entreprise import siae_update_entreprise, siae_update_etablissement, siae_update_exercice - - +# Result for a call to: https://entreprises.api.gouv.fr/api/v3/insee/unites_legales/130025265 MOCK_ENTREPRISE_API_DATA = """ { "data": { @@ -66,31 +56,7 @@ } """ - -class TestSiaeUpdateEntreprise(TestCase): - def setUp(self): - self.siae = SiaeFactory(siret="13002526500013") - - @patch("requests.get") - def test_siae_update_entreprise(self, mock_api): - mock_response = mock_api.return_value - mock_response.json.return_value = json.loads(MOCK_ENTREPRISE_API_DATA) - mock_response.status_code = 200 - - result, entreprise = siae_update_entreprise(self.siae) - - self.siae.refresh_from_db() - - # Assert the result - self.assertEqual(result, 1) - self.assertIsNotNone(entreprise) - - # Assert the updates - self.assertEqual(self.siae.api_entreprise_forme_juridique, "Service central d'un ministère") - self.assertEqual(self.siae.api_entreprise_forme_juridique_code, "7120") - self.assertLess((timezone.now() - self.siae.api_entreprise_entreprise_last_sync_date).total_seconds(), 60) - - +# Result for a call to: https://entreprises.api.gouv.fr/api/v3/insee/sirene/etablissements/30613890001294 MOCK_ETABLISSEMENT_API_DATA = """ { "data": { @@ -194,32 +160,7 @@ def test_siae_update_entreprise(self, mock_api): } """ - -class TestSiaeUpdateEtablissement(TestCase): - def setUp(self): - self.siae = SiaeFactory(siret="30613890001294") - - @patch("requests.get") - def test_siae_update_etablissement(self, mock_api): - mock_response = mock_api.return_value - mock_response.json.return_value = json.loads(MOCK_ETABLISSEMENT_API_DATA) - mock_response.status_code = 200 - - result, etablissement = siae_update_etablissement(self.siae) - - # Assert the result - self.assertEqual(result, 1) - self.assertIsNotNone(etablissement) - - # Assert the updates - self.siae.refresh_from_db() - self.assertEqual(self.siae.siret, "30613890001294") - self.assertEqual(self.siae.api_entreprise_employees, "2 000 à 4 999 salariés") - self.assertEqual(self.siae.api_entreprise_employees_year_reference, "2016") - self.assertEqual(self.siae.api_entreprise_date_constitution, datetime.date(2021, 10, 13)) - self.assertLess((timezone.now() - self.siae.api_entreprise_etablissement_last_sync_date).total_seconds(), 60) - - +# Result for a call to: https://entreprises.api.gouv.fr/api/v3/dgfip/etablissements/30613890001294/chiffres_affaires MOCK_EXERCICES_API_DATA = """ { "data": [ @@ -236,21 +177,3 @@ def test_siae_update_etablissement(self, mock_api): "links": {} } """ - - -class TestSiaeUpdateExercice(TestCase): - def setUp(self): - self.siae = SiaeFactory(siret="30613890001294") - - @patch("requests.get") - def test_siae_update_exercice(self, mock_api): - mock_response = mock_api.return_value - mock_response.json.return_value = json.loads(MOCK_EXERCICES_API_DATA) - mock_response.status_code = 200 - - siae_update_exercice(self.siae) - - self.siae.refresh_from_db() - self.assertEqual(self.siae.api_entreprise_ca, 900001) - self.assertEqual(self.siae.api_entreprise_ca_date_fin_exercice, datetime.date(2015, 12, 1)) - self.assertLess((timezone.now() - self.siae.api_entreprise_exercice_last_sync_date).total_seconds(), 60) From 43c9ba6223ef129f7f80b08a6f79822afcace490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Thu, 19 Dec 2024 17:33:14 +0100 Subject: [PATCH 6/6] make it less complex --- .../commands/update_api_entreprise_fields.py | 117 ++++++++---------- 1 file changed, 52 insertions(+), 65 deletions(-) diff --git a/lemarche/siaes/management/commands/update_api_entreprise_fields.py b/lemarche/siaes/management/commands/update_api_entreprise_fields.py index 9200aa115..754b8bbc1 100644 --- a/lemarche/siaes/management/commands/update_api_entreprise_fields.py +++ b/lemarche/siaes/management/commands/update_api_entreprise_fields.py @@ -61,6 +61,9 @@ def handle(self, *args, **options): if options["limit"]: siae_queryset = siae_queryset[: options["limit"]] + self._update_siae_api_entreprise_fields(siae_queryset, options["scope"]) + + def _update_siae_api_entreprise_fields(self, siae_queryset, scope): results = { "entreprise": {"success": 0, "error": 0}, "etablissement": {"success": 0, "error": 0}, @@ -73,72 +76,56 @@ def handle(self, *args, **options): if (progress % 50) == 0: self.stdout_info(f"{progress}...") - if siae.siret: - update_data = dict() - if options["scope"] in ("all", "entreprise") and siae.api_entreprise_entreprise_last_sync_date is None: - entreprise, error = entreprise_get_or_error(siae.siret[:9], reason=API_ENTREPRISE_REASON) - if error: - results["entreprise"]["error"] += 1 - self.stdout_error(str(error)) - else: - results["entreprise"]["success"] += 1 - if entreprise: - if entreprise.forme_juridique: - update_data["api_entreprise_forme_juridique"] = entreprise.forme_juridique - if entreprise.forme_juridique_code: - update_data["api_entreprise_forme_juridique_code"] = entreprise.forme_juridique_code - update_data["api_entreprise_entreprise_last_sync_date"] = timezone.now() - - if ( - options["scope"] in ("all", "etablissement") - and siae.api_entreprise_etablissement_last_sync_date is None - ): - etablissement, error = etablissement_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) - if error: - results["etablissement"]["error"] += 1 - self.stdout_error(str(error)) - else: - results["etablissement"]["success"] += 1 - if etablissement: - if etablissement.employees: - update_data["api_entreprise_employees"] = ( - etablissement.employees - if (etablissement.employees != "Unités non employeuses") - else "Non renseigné" - ) - if etablissement.employees_date_reference: - update_data[ - "api_entreprise_employees_year_reference" - ] = etablissement.employees_date_reference - if etablissement.date_constitution: - update_data["api_entreprise_date_constitution"] = etablissement.date_constitution - - update_data["api_entreprise_etablissement_last_sync_date"] = timezone.now() - - if options["scope"] in ("all", "exercice") and siae.api_entreprise_exercice_last_sync_date is None: - exercice, error = exercice_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) - if error: - results["exercice"]["error"] += 1 - self.stdout_error(str(error)) - else: - results["exercice"]["success"] += 1 - if exercice: - if exercice.chiffre_affaires: - update_data["api_entreprise_ca"] = exercice.chiffre_affaires - if exercice.date_fin_exercice: - update_data["api_entreprise_ca_date_fin_exercice"] = datetime.strptime( - exercice.date_fin_exercice, "%Y-%m-%d" - ).date() - - update_data["api_entreprise_exercice_last_sync_date"] = timezone.now() - - Siae.objects.filter(id=siae.id).update(**update_data) - - # small delay to avoid going above the API limitation - # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints" - time.sleep(0.5) - else: + if not siae.siret: self.stdout_error(f"SIAE {siae.id} without SIRET") + continue + + update_data = dict() + if scope in ("all", "entreprise") and siae.api_entreprise_entreprise_last_sync_date is None: + entreprise, error = entreprise_get_or_error(siae.siret[:9], reason=API_ENTREPRISE_REASON) + if error: + results["entreprise"]["error"] += 1 + self.stdout_error(str(error)) + else: + results["entreprise"]["success"] += 1 + update_data["api_entreprise_forme_juridique"] = entreprise.forme_juridique + update_data["api_entreprise_forme_juridique_code"] = entreprise.forme_juridique_code + update_data["api_entreprise_entreprise_last_sync_date"] = timezone.now() + + if scope in ("all", "etablissement") and siae.api_entreprise_etablissement_last_sync_date is None: + etablissement, error = etablissement_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) + if error: + results["etablissement"]["error"] += 1 + self.stdout_error(str(error)) + else: + results["etablissement"]["success"] += 1 + update_data["api_entreprise_employees"] = ( + etablissement.employees + if (etablissement.employees != "Unités non employeuses") + else "Non renseigné" + ) + update_data["api_entreprise_employees_year_reference"] = etablissement.employees_date_reference + update_data["api_entreprise_date_constitution"] = etablissement.date_constitution + update_data["api_entreprise_etablissement_last_sync_date"] = timezone.now() + + if scope in ("all", "exercice") and siae.api_entreprise_exercice_last_sync_date is None: + exercice, error = exercice_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) + if error: + results["exercice"]["error"] += 1 + self.stdout_error(str(error)) + else: + results["exercice"]["success"] += 1 + update_data["api_entreprise_ca"] = exercice.chiffre_affaires + update_data["api_entreprise_ca_date_fin_exercice"] = datetime.strptime( + exercice.date_fin_exercice, "%Y-%m-%d" + ).date() + update_data["api_entreprise_exercice_last_sync_date"] = timezone.now() + + Siae.objects.filter(id=siae.id).update(**update_data) + + # small delay to avoid going above the API limitation, one loop generates 3 requests + # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints" + time.sleep(1) msg_success = [ "----- Synchronisation API Entreprise -----",