diff --git a/NAMESPACE b/NAMESPACE index 69fe2334..356898c1 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -10,6 +10,7 @@ export(get_closest_matches) export(get_non_matched_elements) export(map_ds_spatial_units) export(map_spatial) +export(read_write_metadata_tables) export(spatial_mapping_country) export(stattab_find_match) export(stattab_make_pattern) diff --git a/R/metadata_tables.R b/R/metadata_tables.R new file mode 100644 index 00000000..195ff3b8 --- /dev/null +++ b/R/metadata_tables.R @@ -0,0 +1,80 @@ +#' Read/Write metadata tables as csv +#' +#' @param ds the dataset: it is expected to have +#' ds$postgres_export +#' the language is to be expected the language +#' the ds is in (one language per pipeline) +#' @param overwrite default is FALSE, when set to TRUE +#' the metadata file will be overwritten +#' +#' @return metadata a tibble containing the metadata that +#' matches what is on file +#' the function watches for files: +#' metadata_tables.csv +#' metadata_table_columns.csv +#' If they are not there or overwrite is set +#' to TRUE templates for these files will be +#' set up. Otherwise the files will be read +#' and reported back +#' @export +#' +#' @examples +#' \dontrun{ +#' # command to call when the metadata should be read or +#' # created: it will be created if it does not exist yet +#' read_write_metadata_tables(ds) +#' +#' # command to call when the metadata should be overwritten +#' read_write_metadata_tables(ds, overwrite = TRUE) +#' } +read_write_metadata_tables <- function(ds, overwrite = FALSE) { + + # set path for metadata storage + path_table <- paste0(ds$dir, "metadata_tables.csv") + path_table_columns <- paste0(ds$dir, "metadata_table_columns.csv") + + if (file.exists(path_table) && file.exists(path_table_columns) &&!overwrite) { + metadata_tables = readr::read_csv(path_table) + metadata_table_columns = readr::read_csv(path_table_columns) + return(list(metadata_tables=metadata_tables, + metadata_table_columns=metadata_table_columns)) + } + + # make template for metadata files + + # metadata_tables entry: if possible load metadata + table_description <- "" + if (ds$organization == "bfs" && ds$format == "px") { + bfs_metadata <- BFS::bfs_get_metadata(number_bfs = ds$sheet, language = ds$lang) + table_description <- unique(bfs_metadata$title) + } + metadata_tables <- tibble::tibble( + name = ds$name, + language = ds$lang, + description = table_description + ) + + # columns are received from the columns of the postgres export + columns <- colnames(ds$postgres_export) + columns <- columns[!(columns %in% c("jahr", "spatialunit_uid"))] + columns_count <- length(columns) + metadata_table_columns <- tibble::tibble( + name = columns, + table_name = rep(ds$name, times = columns_count), + data_type = rep("", times = columns_count), + description = rep("", times = columns_count) + ) + + # write the tibble to a file + write.csv(metadata_tables, path_table, row.names = FALSE, quote = FALSE) + write.csv(metadata_table_columns, path_table_columns, row.names = FALSE, quote = FALSE) + print( + paste( + "metadata templates written to\n", + ds$dir, + "\nPlease complete metadata for pipeline." + ) + ) + return(list(metadata_tables=metadata_tables, + metadata_table_columns=metadata_table_columns)) +} diff --git a/man/read_write_metadata_tables.Rd b/man/read_write_metadata_tables.Rd new file mode 100644 index 00000000..027c33e6 --- /dev/null +++ b/man/read_write_metadata_tables.Rd @@ -0,0 +1,41 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/metadata_tables.R +\name{read_write_metadata_tables} +\alias{read_write_metadata_tables} +\title{Read/Write metadata tables as csv} +\usage{ +read_write_metadata_tables(ds, overwrite = FALSE) +} +\arguments{ +\item{ds}{the dataset: it is expected to have +ds$postgres_export +the language is to be expected the language +the ds is in (one language per pipeline)} + +\item{overwrite}{default is FALSE, when set to TRUE +the metadata file will be overwritten} +} +\value{ +metadata a tibble containing the metadata that +matches what is on file +the function watches for files: +metadata_tables.csv +metadata_table_columns.csv +If they are not there or overwrite is set +to TRUE templates for these files will be +set up. Otherwise the files will be read +and reported back +} +\description{ +Read/Write metadata tables as csv +} +\examples{ +\dontrun{ + # command to call when the metadata should be read or + # created: it will be created if it does not exist yet + read_write_metadata_tables(ds) + + # command to call when the metadata should be overwritten + read_write_metadata_tables(ds, overwrite = TRUE) +} +} diff --git a/pipelines/A6/A6.R b/pipelines/A6/A6.R index 116ba75a..b02b0e4d 100644 --- a/pipelines/A6/A6.R +++ b/pipelines/A6/A6.R @@ -36,8 +36,8 @@ ds$postgres_export # ------------------------------------------------------------------------- # Step: Testrun queries on sqllite -# input: ds$postgres_export, A6/queries.sql -# output: A6/queries.log +# input: ds$postgres_export, pipelines/A6/queries.sql +# output: pipelines/A6/queries.log # ------------------------------------------------------------------------- statbotData::testrun_queries( @@ -45,3 +45,11 @@ statbotData::testrun_queries( ds$dir, ds$name ) + +# ------------------------------------------------------------------------- +# Step: Write metadata tables +# input: ds$postgres_export +# output: pipelines/A6/metadata.csv +# ------------------------------------------------------------------------- + +read_write_metadata_tables(ds) diff --git a/pipelines/A6/metadata_table_columns.csv b/pipelines/A6/metadata_table_columns.csv new file mode 100644 index 00000000..dabb903b --- /dev/null +++ b/pipelines/A6/metadata_table_columns.csv @@ -0,0 +1,3 @@ +name,table_name,data_type,description +thema,abstimmungsvorlagen_seit_1971,, +anzahl_abstimmungsvorlagen,abstimmungsvorlagen_seit_1971,, diff --git a/pipelines/A6/metadata_tables.csv b/pipelines/A6/metadata_tables.csv new file mode 100644 index 00000000..b7f96bc9 --- /dev/null +++ b/pipelines/A6/metadata_tables.csv @@ -0,0 +1,2 @@ +name,language,description +abstimmungsvorlagen_seit_1971,de,Volksabstimmungen (Anzahl Vorlagen nach Thema seit 1971)