Skip to content

Commit

Permalink
Import - MAJIC: détection automatique des fichiers dans le répertoire
Browse files Browse the repository at this point in the history
  • Loading branch information
@mdouchin
mdouchin committed Jan 21, 2025
1 parent 9a559ce commit 8fd4df5
Show file tree
Hide file tree
Showing 5 changed files with 273 additions and 154 deletions.
294 changes: 180 additions & 114 deletions cadastre/cadastre_import.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,36 +102,42 @@ def __init__(self, dialog):

s = QSettings()
self.majicSourceFileNames = [
{'key': '[FICHIER_BATI]',
'value': s.value("cadastre/batiFileName", 'REVBATI.800', type=str),
'table': 'bati',
'required': True
},
{'key': '[FICHIER_FANTOIR]',
'value': s.value("cadastre/fantoirFileName", 'TOPFANR.800', type=str),
'table': 'fanr',
'required': True
},
{'key': '[FICHIER_LOTLOCAL]',
'value': s.value("cadastre/lotlocalFileName", 'REVD166.800', type=str),
'table': 'lloc',
'required': False
},
{'key': '[FICHIER_NBATI]',
'value': s.value("cadastre/nbatiFileName", 'REVNBAT.800', type=str),
'table': 'nbat',
'required': True
},
{'key': '[FICHIER_PDL]',
'value': s.value("cadastre/pdlFileName", 'REVFPDL.800', type=str),
'table': 'pdll',
'required': False
},
{'key': '[FICHIER_PROP]',
'value': s.value("cadastre/propFileName", 'REVPROP.800', type=str),
'table': 'prop',
'required': True
}
{
'key': '[FICHIER_BATI]',
'regex': s.value("cadastre/batiFileName", 'BATI', type=str),
'table': 'bati',
'required': True
},
{
'key': '[FICHIER_FANTOIR]',
'value': s.value("cadastre/fantoirFileName", 'FANTOIR|FANR', type=str),
'table': 'fanr',
'required': True
},
{
'key': '[FICHIER_LOTLOCAL]',
'regex': s.value("cadastre/lotlocalFileName", 'LLOC|D166', type=str),
'table': 'lloc',
'required': False
},
{
'key': '[FICHIER_NBATI]',
'regex': s.value("cadastre/nbatiFileName", 'NBAT', type=str),
'table': 'nbat',
'required': True
},
{
'key': '[FICHIER_PDL]',
'regex': s.value("cadastre/pdlFileName", 'PDL', type=str),
'table': 'pdll',
'required': False
},
{
'key': '[FICHIER_PROP]',
'regex': s.value("cadastre/propFileName", 'PROP.txt', type=str),
'table': 'prop',
'required': True
},
]

if self.dialog.dbType == 'postgis':
Expand Down Expand Up @@ -329,7 +335,7 @@ def importMajic(self):
# Import MAJIC files into database
# No use of COPY FROM to allow import into distant databases
importScript = {
'title': 'Import des fichiers majic',
'title': 'Import des fichiers MAJIC',
'method': self.import_majic_into_database
}
scriptList.append(importScript)
Expand Down Expand Up @@ -425,44 +431,24 @@ def importMajic(self):

return None

def chunk(self, iterable, n=100000, padvalue=None):
"""
Chunks an iterable (file, etc.)
into pieces
"""
from itertools import zip_longest
return zip_longest(*[iter(iterable)] * n, fillvalue=padvalue)

def import_majic_into_database(self) -> bool:
def get_available_majic_files(self) -> tuple[dict, dict]:
"""
Method which read each majic file
and bulk import data into temp tables
Returns False if no file processed
Get the list of MAJIC files to import
"""
processed_files_count = 0
majic_files_key = []
majic_files_found = {}

# Regex to remove all chars not in the range in ASCII table from space to ~
# http://www.catonmat.net/blog/my-favorite-regex/
r = re.compile(r"[^ -~]")

# Loop through all majic files

# 1st path to build the complet liste for each majic source type (nbat, bati, lloc, etc.)
# and read 1st line to get departement and direction to compare to inputs
dep_dirs = {}
for item in self.majicSourceFileNames:
table = item['table']
value = item['value']
# Get majic files for item
file_regex = item['regex']
# Get MAJIC files for item
maj_list = []
for root, dirs, files in os.walk(self.dialog.majicSourceDir):
for i in files:
if os.path.split(i)[1] == value:
file_path = os.path.join(root, i)
for file_sub_path in files:
# self.qc.updateLog(file_sub_path)
# Check if the file matches the regex given for this file type
if re.search(file_regex, os.path.split(file_sub_path)[1].upper()):
# Add file path to the list
file_path = os.path.join(root, file_sub_path)
maj_list.append(file_path)

# Store dep_dir for this file
Expand All @@ -481,23 +467,28 @@ def import_majic_into_database(self) -> bool:

majic_files_found[table] = maj_list

# Check if some important majic files are missing
return dep_dirs, majic_files_found

def check_missing_majic_files(self, majic_files_found: dict) -> bool:
"""
Check if the mandatory MAJIC files have been found in the directory
"""
f_keys = [a for a in majic_files_found if majic_files_found[a]]
r_keys = [a['table'] for a in self.majicSourceFileNames if a['required']]
m_keys = [a for a in r_keys if a not in f_keys]
if m_keys:
missing_files = [a for a in r_keys if a not in f_keys]
if missing_files:
msg = (
"<b>Des fichiers MAJIC importants sont manquants: {} </b><br/>"
"Vérifier le chemin des fichiers MAJIC:<br/>"
"{} <br/>"
"ainsi que les noms des fichiers configurés dans les options du plugin Cadastre:<br/>"
"{}<br/><br/>"
"Vous pouvez télécharger les fichiers fantoirs à cette adresse :<br/>"
"<b>Des fichiers MAJIC importants sont manquants : {} </b><br/>"
"Vérifier le chemin des fichiers MAJIC :<br/>"
"<b>{}</b> <br/>"
"ainsi que les mots recherchés pour chaque type de fichier configurés dans les options du plugin Cadastre :<br/>"
"<b>{}</b><br/><br/>"
"<b>NB:</b> Vous pouvez télécharger les fichiers FANTOIR à cette adresse :<br/>"
"<a href='{}'>{}</a><br/>"
).format(
', '.join(m_keys),
', '.join(missing_files),
self.dialog.majicSourceDir,
', '.join([a['value'].upper() for a in self.majicSourceFileNames]),
', '.join([a['regex'].upper() for a in self.majicSourceFileNames]),
URL_FANTOIR,
URL_FANTOIR,
)
Expand All @@ -512,6 +503,14 @@ def import_majic_into_database(self) -> bool:
self.qc.updateLog(msg)
return False

return True

def check_majic_departement_direction(self, dep_dirs: dict) -> bool:
"""
Check if departement and direction are the same for every MAJIC file.
Check if departement and direction are different from those
given by the user in dialog
"""
# Check if departement and direction are the same for every file
if len(list(dep_dirs.keys())) > 1:
self.go = False
Expand All @@ -523,6 +522,7 @@ def import_majic_into_database(self) -> bool:
"</b>\n<br/> {}".format(lst)
)
self.qc.updateLog("<b>Veuillez réaliser l'import en %s fois.</b>" % len(list(dep_dirs.keys())))

return False

# Check if departement and direction are different from those given by the user in dialog
Expand Down Expand Up @@ -557,8 +557,114 @@ def import_majic_into_database(self) -> bool:
else:
self.go = False
self.qc.updateLog(msg)

return False

return True

def chunk(self, iterable, n=100000, padvalue=None):
"""
Chunks an iterable (file, etc.)
into pieces
"""
from itertools import zip_longest
return zip_longest(*[iter(iterable)] * n, fillvalue=padvalue)

def import_majic_file_into_database(self, table_name: str, file_path: str, dep_dir: str) -> bool:
"""
Import a single MAJIC file into the corresponding database table
For example, import a file REVPROP.txt into the "prop" table
"""
# Regex to remove all chars not in the range in ASCII table from space to ~
# http://www.catonmat.net/blog/my-favorite-regex/
regex_remove_non_ascii = re.compile(r"[^ -~]")

# read file content
with open(file_path, encoding='ascii', errors='replace') as fin:
# Divide file into chunks
for a in self.chunk(fin, self.maxInsertRows):
# Build sql INSERT query depending on database
if self.dialog.dbType == 'postgis':
try:
sql = "BEGIN;"
sql = CadastreCommon.setSearchPath(sql, self.dialog.schema)
# Build INSERT list
sql += '\n'.join(
[
"INSERT INTO \"{}\" VALUES ({});".format(
table_name,
self.connector.quoteString(
regex_remove_non_ascii.sub(' ', x.strip('\r\n'))
)
) for x in a if x and x[0:3] == dep_dir
]
)
sql += "COMMIT;"
except MemoryError:
# Issue #326
self.qc.updateLog(
"<b>"
"ERREUR : Mémoire - Veuillez recommencer l'import en changeant le "
"paramètre 'Taille maximum des requêtes INSERT' selon la "
"documentation : {}{}"
"</b>".format(
URL_DOCUMENTATION,
"/extension-qgis/configuration/#performances",
)
)
self.go = False

return False

self.executeSqlQuery(sql)
else:
try:
c = self.connector._get_cursor()
c.executemany(
f'INSERT INTO "{table_name}" VALUES (?)',
[(regex_remove_non_ascii.sub(' ', x.strip('\r\n')),) for x in a if x and x[0:3] == dep_dir])
self.connector._commit()
c.close()
del c
except:
self.qc.updateLog(
"<b>"
f"ERREUR : l'import du fichier '{file_path}' a échoué"
"</b>"
)
self.go = False

return False

return True

def import_majic_into_database(self) -> bool:
"""
Method which read each majic file
and bulk import data into temp tables
Returns False if no file processed
"""
processed_files_count = 0
majic_files_key = []

# Loop through all majic files

# 1st path to build the complet list for each majic source type (nbat, bati, lloc, etc.)
# and read 1st line to get departement and direction to compare to inputs
dep_dirs, majic_files_found = self.get_available_majic_files()

# Check if some important majic files are missing
check_missing = self.check_missing_majic_files(majic_files_found)
if not check_missing:
return False

# Check departement & direction
check_depdir = self.check_majic_departement_direction(dep_dirs)
if not check_depdir:
return False

# 2nd path to insert data
dep_dir = f'{self.dialog.edigeoDepartement}{self.dialog.edigeoDirection}'
for item in self.majicSourceFileNames:
Expand All @@ -568,49 +674,9 @@ def import_majic_into_database(self) -> bool:
for file_path in majic_files_found[table]:
self.qc.updateLog(file_path)

# read file content
with open(file_path, encoding='ascii', errors='replace') as fin:
# Divide file into chunks
for a in self.chunk(fin, self.maxInsertRows):
# Build sql INSERT query depending on database
if self.dialog.dbType == 'postgis':
try:
sql = "BEGIN;"
sql = CadastreCommon.setSearchPath(sql, self.dialog.schema)
# Build INSERT list
sql += '\n'.join(
[
"INSERT INTO \"{}\" VALUES ({});".format(
table,
self.connector.quoteString(r.sub(' ', x.strip('\r\n')))
) for x in a if x and x[0:3] == dep_dir
]
)
sql += "COMMIT;"
except MemoryError:
# Issue #326
self.qc.updateLog(
"<b>"
"ERREUR : Mémoire - Veuillez recommencer l'import en changeant le "
"paramètre 'Taille maximum des requêtes INSERT' selon la "
"documentation : {}{}"
"</b>".format(
URL_DOCUMENTATION,
"/extension-qgis/configuration/#performances",
)
)
self.go = False
return False

self.executeSqlQuery(sql)
else:
c = self.connector._get_cursor()
c.executemany(
f'INSERT INTO {table} VALUES (?)',
[(r.sub(' ', x.strip('\r\n')),) for x in a if x and x[0:3] == dep_dir])
self.connector._commit()
c.close()
del c
import_file = self.import_majic_file_into_database(table, file_path, dep_dir)
if not import_file:
continue

if not processed_files_count:
self.qc.updateLog(
Expand Down
2 changes: 1 addition & 1 deletion cadastre/dialogs/dialog_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ def check_database_for_existing_structure(self):
self.dialog.hasMajicData = has_majic_data
self.dialog.hasMajicDataParcelle = has_majic_data_parcelle
self.dialog.hasMajicDataProp = has_majic_data_prop
self.dialog.hasMajicData = has_majic_data_voie
self.dialog.hasMajicDataVoie = has_majic_data_voie

def checkDatabaseForExistingTable(self, tableName, schemaName=''):
"""
Expand Down
5 changes: 4 additions & 1 deletion cadastre/dialogs/import_dialog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,10 +180,13 @@ def chooseDataPath(self, key):
Ask the user to select a folder
and write down the path to appropriate field
"""
root_directory = str(self.pathSelectors[key]['input'].text()).strip(' \t')
if not root_directory:
root_directory = os.path.expanduser("~")
ipath = QFileDialog.getExistingDirectory(
None,
"Choisir le répertoire contenant les fichiers",
str(self.pathSelectors[key]['input'].text().encode('utf-8')).strip(' \t')
root_directory
)
if os.path.exists(str(ipath)):
self.pathSelectors[key]['input'].setText(str(ipath))
Expand Down
Loading

0 comments on commit 8fd4df5

Please sign in to comment.