Skip to content

Commit

Permalink
replace mocked tests by endpoint tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@luabida
luabida committed Nov 25, 2024
1 parent e3bd000 commit 7d42a14
Show file tree
Hide file tree
Showing 31 changed files with 644,618 additions and 919 deletions.
895 changes: 895 additions & 0 deletions pysus/Notebooks/CNES.ipynb
View file

Large diffs are not rendered by default.

703 changes: 703 additions & 0 deletions pysus/Notebooks/Chikungunya.ipynb
View file

Large diffs are not rendered by default.

497 changes: 497 additions & 0 deletions pysus/Notebooks/Dengue.ipynb
View file

Large diffs are not rendered by default.

2,910 changes: 2,910 additions & 0 deletions pysus/Notebooks/IBGE_data.ipynb
View file

Large diffs are not rendered by default.

500 changes: 500 additions & 0 deletions pysus/Notebooks/Infodengue.ipynb
View file

Large diffs are not rendered by default.

1,526 changes: 1,526 additions & 0 deletions pysus/Notebooks/Infogripe.ipynb
View file

Large diffs are not rendered by default.

173 changes: 173 additions & 0 deletions pysus/Notebooks/Preprocessing SIM with municipality.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from pysus.preprocessing.geodata import add_data_to_municipality\n",
"from pysus.preprocessing.SIM import group_and_count, redistribute_missing, redistribute_cid_chapter\n",
"from pysus.preprocessing.decoders import translate_variables_SIM\n",
"\n",
"from pysus.ftp.databases.sim import SIM\n",
"\n",
"sim = SIM().load()\n",
"\n",
"states = [\"AC\", \"AM\", \"AP\", \"PA\", \"RO\", \"RR\", \"TO\"]\n",
"variables = ['CODMUNRES','SEXO','IDADE_ANOS','CID10_CHAPTER']\n",
"filter_variables = variables[:3]\n",
"geo_df = pd.DataFrame()\n",
"ages_args = {\n",
" \"start\": 0,\n",
" \"end\": 90,\n",
" \"freq\": 5\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 330k/330k [00:00<00:00, 167MB/s]\n",
"2023-09-19 13:53:06.107 | DEBUG | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:06.128 | DEBUG | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:06.129 | INFO | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
"2023-09-19 13:53:06.371 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:06.392 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:06.393 | INFO | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████| 1.42M/1.42M [00:00<00:00, 1.07GB/s]\n",
"2023-09-19 13:53:08.409 | DEBUG | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:08.431 | DEBUG | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:08.432 | INFO | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
"2023-09-19 13:53:08.879 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:08.899 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:08.899 | INFO | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
"100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 264k/264k [00:00<00:00, 173MB/s]\n",
"2023-09-19 13:53:11.067 | DEBUG | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:11.088 | DEBUG | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:11.089 | INFO | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
"2023-09-19 13:53:11.311 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:11.332 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:11.333 | INFO | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████| 3.22M/3.22M [00:00<00:00, 4.17GB/s]\n",
"2023-09-19 13:53:13.178 | DEBUG | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:13.199 | DEBUG | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:13.200 | INFO | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
"2023-09-19 13:53:14.100 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:14.122 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:14.123 | INFO | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
"100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 665k/665k [00:00<00:00, 509MB/s]\n",
"2023-09-19 13:53:19.139 | DEBUG | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:19.160 | DEBUG | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:19.161 | INFO | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
"2023-09-19 13:53:19.459 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:19.481 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:19.482 | INFO | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
"100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 230k/230k [00:00<00:00, 146MB/s]\n",
"2023-09-19 13:53:21.358 | DEBUG | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:21.379 | DEBUG | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:21.381 | INFO | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
"2023-09-19 13:53:21.597 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:21.618 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:21.619 | INFO | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
"100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 659k/659k [00:00<00:00, 554MB/s]\n",
"2023-09-19 13:53:22.812 | DEBUG | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:22.833 | DEBUG | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:22.834 | INFO | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
"2023-09-19 13:53:23.138 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
"220 Microsoft FTP Service\n",
"2023-09-19 13:53:23.159 | DEBUG | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
"2023-09-19 13:53:23.161 | INFO | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n"
]
}
],
"source": [
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"for state in states:\n",
" df = sim.download(sim.get_files(\"CID10\",state,2018)).to_dataframe()\n",
" df = translate_variables_SIM(df,age_classes=True,classify_args=ages_args,classify_cid10_chapters=True)\n",
" df = df[variables]\n",
" counts = group_and_count(df,variables)\n",
" counts = redistribute_missing(counts,filter_variables)\n",
" counts = redistribute_cid_chapter(counts,filter_variables)\n",
" geo_df = geo_df.append(add_data_to_municipality(counts,title_cols=['SEXO','IDADE_ANOS','CID10_CHAPTER'], nan_string='NA'))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Index(['code_muni', 'name_muni', 'code_state', 'abbrev_state', 'name_state',\n",
" 'code_region', 'name_region', 'geometry', 'Feminino-[0.0, 5.0)-1',\n",
" 'Feminino-[0.0, 5.0)-2',\n",
" ...\n",
" 'NA-[50.0, 55.0)-8', 'NA-[55.0, 60.0)-8', 'NA-[60.0, 65.0)-8',\n",
" 'NA-[65.0, 70.0)-8', 'NA-[70.0, 75.0)-8', 'NA-[75.0, 80.0)-8',\n",
" 'NA-[80.0, 85.0)-8', 'NA-[85.0, 90.0)-8', 'NA-[90.0, inf)-8',\n",
" 'NA-NA-8'],\n",
" dtype='object', length=1088)"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"geo_df.columns"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.5"
},
"vscode": {
"interpreter": {
"hash": "2a96a5ccec8dfcba7d06b2e71f6eef3b5dac5716461bf5d73ea1bb7ee462cdaa"
}
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Loading

0 comments on commit 7d42a14

Please sign in to comment.