From 346c8a21a55c221f7175a09ca25d80ef1290d21a Mon Sep 17 00:00:00 2001 From: Martin Landa Date: Thu, 14 Nov 2024 16:58:23 +0100 Subject: [PATCH] docs: Generate manual from markdown using mkdocs (#3849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Markus Neteler Co-authored-by: Edouard Choinière <27212526+echoix@users.noreply.github.com> Co-authored-by: Markus Neteler Co-authored-by: Vaclav Petras --- .flake8 | 1 + REQUIREMENTS.md | 2 + gui/scripts/Makefile | 2 +- include/Make/Grass.make | 1 + include/Make/GuiScript.make | 18 +- include/Make/Html.make | 9 +- include/Make/HtmlRules.make | 16 + include/Make/NoHtml.make | 3 + include/Make/Rules.make | 2 +- lib/gis/parser_rest_md.c | 85 ++--- lib/init/Makefile | 3 +- man/Makefile | 92 ++++- man/build.py | 157 ++++++++ man/build_check.py | 9 +- man/build_class.py | 133 ++++--- man/build_class_graphical.py | 21 +- man/build_full_index.py | 171 +++++---- man/build_graphical_index.py | 125 +++---- man/build_html.py | 184 +++------ man/build_index.py | 48 ++- man/build_keywords.py | 249 +++++++------ man/build_manual_gallery.py | 105 ++++-- man/build_md.py | 266 +++++++++++++ man/build_topics.py | 199 ++++++---- man/mkdocs/grassdocs.css | 21 ++ man/mkdocs/mkdocs.yml | 43 +++ man/mkdocs/overrides/partials/footer.html | 100 +++++ man/mkdocs/requirements.txt | 5 + man/parser_standard_options.py | 75 ++-- raster/Makefile | 4 + utils/Makefile | 8 +- utils/mkdocs.py | 435 ++++++++++++++++++++++ utils/mkhtml.py | 426 +-------------------- utils/mkmarkdown.py | 167 +++++++++ 34 files changed, 2089 insertions(+), 1096 deletions(-) create mode 100644 man/build.py create mode 100644 man/build_md.py create mode 100644 man/mkdocs/grassdocs.css create mode 100644 man/mkdocs/mkdocs.yml create mode 100644 man/mkdocs/overrides/partials/footer.html create mode 100644 man/mkdocs/requirements.txt create mode 100644 utils/mkdocs.py create mode 100644 utils/mkmarkdown.py diff --git a/.flake8 b/.flake8 index 8026eb7073e..fc8dd9cb65e 100644 --- a/.flake8 +++ b/.flake8 @@ -20,6 +20,7 @@ per-file-ignores = # E741 ambiguous variable name 'l' __init__.py: F403 man/build_html.py: E501 + man/build_md.py: E501 doc/python/m.distance.py: E501 gui/scripts/d.wms.py: E501 gui/wxpython/image2target/g.gui.image2target.py: E501 diff --git a/REQUIREMENTS.md b/REQUIREMENTS.md index 5d173133716..ff13e49331e 100644 --- a/REQUIREMENTS.md +++ b/REQUIREMENTS.md @@ -35,6 +35,8 @@ for other platforms you may have to install some of them. GDAL: [https://gdal.org](https://gdal.org) - **Python >= 3.8** (for temporal framework, scripts, wxGUI, and ctypes interface) [https://www.python.org](https://www.python.org) +- **MkDocs** with "Material" theme Python packages for the manual pages: + See `man/mkdocs/requirements.txt`. ## Optional packages diff --git a/gui/scripts/Makefile b/gui/scripts/Makefile index 7630fd6d8f5..56c8a27f8e7 100644 --- a/gui/scripts/Makefile +++ b/gui/scripts/Makefile @@ -1,7 +1,7 @@ MODULE_TOPDIR = ../.. -include $(MODULE_TOPDIR)/include/Make/Rules.make include $(MODULE_TOPDIR)/include/Make/Vars.make +include $(MODULE_TOPDIR)/include/Make/Rules.make include $(MODULE_TOPDIR)/include/Make/Python.make DSTDIR = $(GUIDIR)/scripts diff --git a/include/Make/Grass.make b/include/Make/Grass.make index 137a45d1955..17ce25fcb71 100644 --- a/include/Make/Grass.make +++ b/include/Make/Grass.make @@ -59,6 +59,7 @@ DOCSDIR = $(ARCH_DISTDIR)/docs ETC = $(ARCH_DISTDIR)/etc GUIDIR = $(ARCH_DISTDIR)/gui HTMLDIR = $(ARCH_DISTDIR)/docs/html +MDDIR = $(ARCH_DISTDIR)/docs/mkdocs SCRIPTDIR = $(ARCH_DISTDIR)/scripts MSG_DIR = $(ARCH_DISTDIR)/etc/msgs MO_DIR = $(ARCH_DISTDIR)/locale diff --git a/include/Make/GuiScript.make b/include/Make/GuiScript.make index dbbc7609882..7451e2b693d 100644 --- a/include/Make/GuiScript.make +++ b/include/Make/GuiScript.make @@ -10,6 +10,8 @@ include $(MODULE_TOPDIR)/include/Make/HtmlRules.make MODULES := $(patsubst g.gui.%.py,%,$(wildcard g.gui.*.py)) CMDHTML := $(patsubst %,$(HTMLDIR)/g.gui.%.html,$(MODULES)) GUIHTML := $(patsubst %,$(HTMLDIR)/wxGUI.%.html,$(MODULES)) +CMDMAN := $(patsubst %,$(MDDIR)/source/g.gui.%.md,$(MODULES)) +GUIMAN := $(patsubst %,$(MDDIR)/source/wxGUI.%.md,$(MODULES)) ifdef MINGW SCRIPTEXT = .py BATFILES := $(patsubst %,$(BIN)/g.gui.%.bat,$(MODULES)) @@ -19,26 +21,40 @@ BATFILES = endif PYFILES := $(patsubst %,$(SCRIPTDIR)/g.gui.%$(SCRIPTEXT),$(MODULES)) -guiscript: $(IMGDST) $(PYFILES) $(BATFILES) +guiscript: $(IMGDST) $(IMGDST_MD) $(PYFILES) $(BATFILES) # we cannot use cross-compiled g.parser for generating html files ifndef CROSS_COMPILING $(MAKE) $(CMDHTML) -rm -f g.gui.*.tmp.html $(MAKE) $(GUIHTML) +# $(MAKE) $(CMDMAN) +# $(MAKE) $(GUIMAN) endif $(HTMLDIR)/g.gui.%.html: g.gui.%.html g.gui.%.tmp.html | $(HTMLDIR) VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) MODULE_TOPDIR=$(MODULE_TOPDIR) \ $(PYTHON) $(GISBASE)/utils/mkhtml.py g.gui.$* $(GRASS_VERSION_DATE) > $@ +$(MDDIR)/source/g.gui.%.md: g.gui.%.md g.gui.%.tmp.md | $(MDDIR) + VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) MODULE_TOPDIR=$(MODULE_TOPDIR) \ + $(PYTHON) $(GISBASE)/utils/mkmarkdown.py g.gui.$* $(GRASS_VERSION_DATE) > $@ + $(HTMLDIR)/wxGUI.%.html: g.gui.%.html | $(HTMLDIR) -rm -f g.gui.$*.tmp.html VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) MODULE_TOPDIR=$(MODULE_TOPDIR) \ $(PYTHON) $(GISBASE)/utils/mkhtml.py g.gui.$* $(GRASS_VERSION_DATE) > $@ +$(MDDIR)/source/wxGUI.%.md: g.gui.%.md | $(MDDIR) + -rm -f g.gui.$*.tmp.md + VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) MODULE_TOPDIR=$(MODULE_TOPDIR) \ + $(PYTHON) $(GISBASE)/utils/mkmarkdown.py g.gui.$* $(GRASS_VERSION_DATE) > $@ + g.gui.%.tmp.html: $(SCRIPTDIR)/g.gui.% $(call htmldesc,$<,$@) +g.gui.%.tmp.md: $(SCRIPTDIR)/g.gui.% + $(call mddesc,$<,$@) + $(SCRIPTDIR)/g.gui.%$(SCRIPTEXT): g.gui.%.py | $(SCRIPTDIR) $(INSTALL) $< $@ diff --git a/include/Make/Html.make b/include/Make/Html.make index fa736d3405a..335c841e149 100644 --- a/include/Make/Html.make +++ b/include/Make/Html.make @@ -7,19 +7,26 @@ $(HTMLDIR)/%.html: %.html %.tmp.html $(HTMLSRC) $(IMGDST) | $(HTMLDIR) VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) MODULE_TOPDIR=$(MODULE_TOPDIR) \ $(PYTHON) $(GISBASE)/utils/mkhtml.py $* > $@ +$(MDDIR)/source/%.md: %.md %.tmp.md $(HTMLSRC) $(IMGDST_MD) | $(MDDIR) + VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) MODULE_TOPDIR=$(MODULE_TOPDIR) \ + $(PYTHON) $(GISBASE)/utils/mkmarkdown.py $* > $@ + $(MANDIR)/%.$(MANSECT): $(HTMLDIR)/%.html $(HTML2MAN) "$<" "$@" %.tmp.html: $(HTMLSRC) if [ "$(HTMLSRC)" != "" ] ; then $(call htmldesc,$<,$@) ; fi +%.tmp.md: $(HTMLSRC) + if [ "$(HTMLSRC)" != "" ] ; then $(call mddesc,$<,$@) ; fi + ifdef CROSS_COMPILING html: else -html: $(HTMLDIR)/$(PGM).html $(MANDIR)/$(PGM).$(MANSECT) +html: $(HTMLDIR)/$(PGM).html $(MANDIR)/$(PGM).$(MANSECT) # $(MDDIR)/source/$(PGM).md endif diff --git a/include/Make/HtmlRules.make b/include/Make/HtmlRules.make index 0c79a05a2ad..cdeaf317b98 100644 --- a/include/Make/HtmlRules.make +++ b/include/Make/HtmlRules.make @@ -3,13 +3,20 @@ htmldesc = $(call run_grass,$(1) --html-description < /dev/null | grep -v '\|\| ' > $(2)) +mddesc = $(call run_grass,$(1) --md-description < /dev/null > $(2)) + IMGSRC := $(wildcard *.png) $(wildcard *.jpg) $(wildcard *.gif) IMGDST := $(patsubst %,$(HTMLDIR)/%,$(IMGSRC)) +IMGDST_MD := $(patsubst %,$(MDDIR)/source/%,$(IMGSRC)) ifneq ($(strip $(IMGDST)),) .SECONDARY: $(IMGDST) endif +ifneq ($(strip $(IMGDST_MD)),) +.SECONDARY: $(IMGDST_MD) +endif + $(HTMLDIR)/%.png: %.png | $(HTMLDIR) $(INSTALL_DATA) $< $@ @@ -18,3 +25,12 @@ $(HTMLDIR)/%.jpg: %.jpg | $(HTMLDIR) $(HTMLDIR)/%.gif: %.gif | $(HTMLDIR) $(INSTALL_DATA) $< $@ + +$(MDDIR)/source/%.png: %.png | $(MDDIR) + $(INSTALL_DATA) $< $@ + +$(MDDIR)/source/%.jpg: %.jpg | $(MDDIR) + $(INSTALL_DATA) $< $@ + +$(MDDIR)/source/%.gif: %.gif | $(MDDIR) + $(INSTALL_DATA) $< $@ diff --git a/include/Make/NoHtml.make b/include/Make/NoHtml.make index 915d2912c30..5742ff6268f 100644 --- a/include/Make/NoHtml.make +++ b/include/Make/NoHtml.make @@ -2,5 +2,8 @@ $(HTMLDIR)/$(PGM).html: @echo no HTML documentation available +$(MDDIR)/source/$(PGM).md: + @echo no Markdown documentation available + $(MANDIR)/$(PGM).$(MANSECT): @echo no manual page available diff --git a/include/Make/Rules.make b/include/Make/Rules.make index 3681c667a02..9e765fab7a4 100644 --- a/include/Make/Rules.make +++ b/include/Make/Rules.make @@ -6,7 +6,7 @@ first: pre default ARCH_DIRS = $(ARCH_DISTDIR) $(ARCH_BINDIR) $(ARCH_INCDIR) $(ARCH_LIBDIR) \ $(BIN) $(ETC) \ $(DRIVERDIR) $(DBDRIVERDIR) $(FONTDIR) $(DOCSDIR) $(HTMLDIR) \ - $(MANBASEDIR) $(MANDIR) $(UTILSDIR) + $(MDDIR) $(MDDIR)/source $(MANBASEDIR) $(MANDIR) $(UTILSDIR) pre: | $(ARCH_DIRS) diff --git a/lib/gis/parser_rest_md.c b/lib/gis/parser_rest_md.c index 551c72b40e7..c607acc4214 100644 --- a/lib/gis/parser_rest_md.c +++ b/lib/gis/parser_rest_md.c @@ -4,7 +4,7 @@ \brief GIS Library - Argument parsing functions (reStructuredText and Markdown output) - (C) 2012-2023 by the GRASS Development Team + (C) 2012-2024 by the GRASS Development Team This program is free software under the GNU General Public License (>=v2). Read the file COPYING that comes with GRASS for details. @@ -20,6 +20,8 @@ #include "parser_local_proto.h" +#define MD_NEWLINE " " + static void usage_rest_md(bool rest); static void print_flag(const char *key, const char *label, const char *description, bool rest); @@ -41,7 +43,6 @@ void usage_rest_md(bool rest) struct Option *opt; struct Flag *flag; const char *type; - char *header = NULL; int new_prompt = 0; new_prompt = G__uses_new_gisprompt(); @@ -51,35 +52,17 @@ void usage_rest_md(bool rest) if (!st->pgm_name) st->pgm_name = "??"; - /* main header */ - G_asprintf(&header, "%s - GRASS GIS manual", st->pgm_name); - if (rest) { - size_t s; - fprintf(stdout, "%s\n", header); - for (s = 0; s < strlen(header); s++) { - fprintf(stdout, "="); - } - fprintf(stdout, "\n"); - } - else { - fprintf(stdout, "# %s\n", header); - } - fprintf(stdout, "\n"); + /* print metadata used by man/build*.py */ + fprintf(stdout, "---\n"); + fprintf(stdout, "name: %s\n", st->pgm_name); + fprintf(stdout, "description: %s\n", st->module_info.description); + fprintf(stdout, "keywords: "); + G__print_keywords(stdout, NULL, FALSE); + fprintf(stdout, "\n---\n\n"); - /* GRASS GIS logo */ - if (rest) { - fprintf(stdout, ".. image:: grass_logo.png\n"); - fprintf(stdout, " :align: center\n"); - fprintf(stdout, " :alt: GRASS logo\n"); - } - else { - fprintf(stdout, "![GRASS logo](./grass_logo.png)\n"); - } - /* horizontal line */ - fprintf(stdout, "\n---"); - if (rest) - fprintf(stdout, "-"); - fprintf(stdout, "\n\n"); + /* main header */ + if (!rest) + fprintf(stdout, "# %s\n\n", st->pgm_name); /* header - GRASS module */ if (!rest) @@ -88,7 +71,7 @@ void usage_rest_md(bool rest) if (rest) fprintf(stdout, "----"); fprintf(stdout, "\n"); - fprintf(stdout, "**%s**", st->pgm_name); + fprintf(stdout, "***%s***", st->pgm_name); if (st->module_info.label || st->module_info.description) fprintf(stdout, " - "); @@ -130,13 +113,13 @@ void usage_rest_md(bool rest) } fprintf(stdout, "**%s**", st->pgm_name); if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); if (rest) fprintf(stdout, "| "); fprintf(stdout, "**%s --help**", st->pgm_name); if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); if (rest) fprintf(stdout, "| "); @@ -219,7 +202,7 @@ void usage_rest_md(bool rest) while (st->n_flags && flag != NULL) { print_flag(&flag->key, flag->label, flag->description, rest); if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); flag = flag->next_flag; } @@ -228,21 +211,21 @@ void usage_rest_md(bool rest) _("Allow output files to overwrite existing files"), rest); if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); } } print_flag("help", NULL, _("Print usage summary"), rest); if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); print_flag("verbose", NULL, _("Verbose module output"), rest); if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); print_flag("quiet", NULL, _("Quiet module output"), rest); if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); print_flag("ui", NULL, _("Force launching GUI dialog"), rest); fprintf(stdout, "\n"); @@ -263,7 +246,7 @@ void usage_rest_md(bool rest) opt = opt->next_opt; if (opt != NULL) { if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); } fprintf(stdout, "\n"); } @@ -284,7 +267,7 @@ void print_flag(const char *key, const char *label, const char *description, fprintf(stdout, "-"); fprintf(stdout, "-%s**", key); if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); if (label != NULL) { if (rest) @@ -292,13 +275,15 @@ void print_flag(const char *key, const char *label, const char *description, print_escaped(stdout, "\t", rest); print_escaped(stdout, label, rest); if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); } - if (rest) - fprintf(stdout, "| "); - print_escaped(stdout, "\t", rest); - print_escaped(stdout, description, rest); + if (description != NULL) { + if (rest) + fprintf(stdout, "| "); + print_escaped(stdout, "\t", rest); + print_escaped(stdout, description, rest); + } } void print_option(const struct Option *opt, bool rest, char *image_spec_rest) @@ -341,7 +326,7 @@ void print_option(const struct Option *opt, bool rest, char *image_spec_rest) fprintf(stdout, " **[required]**"); } if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); if (opt->label) { if (rest) @@ -352,7 +337,7 @@ void print_option(const struct Option *opt, bool rest, char *image_spec_rest) if (opt->description) { if (opt->label) { if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); } if (rest) @@ -363,7 +348,7 @@ void print_option(const struct Option *opt, bool rest, char *image_spec_rest) if (opt->options) { if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); if (rest) fprintf(stdout, "| "); @@ -375,7 +360,7 @@ void print_option(const struct Option *opt, bool rest, char *image_spec_rest) if (opt->def) { if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); if (rest) fprintf(stdout, "| "); @@ -394,7 +379,7 @@ void print_option(const struct Option *opt, bool rest, char *image_spec_rest) while (opt->opts[i]) { if (opt->descs[i]) { if (!rest) - fprintf(stdout, "\\"); + fprintf(stdout, MD_NEWLINE); fprintf(stdout, "\n"); char *thumbnails = NULL; if (opt->gisprompt) { diff --git a/lib/init/Makefile b/lib/init/Makefile index f6983faada3..2db853fce95 100644 --- a/lib/init/Makefile +++ b/lib/init/Makefile @@ -47,8 +47,9 @@ ifneq ($(strip $(MINGW)),) endif HTMLFILES := $(wildcard *.html) +MDFILES := $(wildcard *.md) -default: $(FILES) $(patsubst %,$(HTMLDIR)/%,$(HTMLFILES)) +default: $(FILES) $(patsubst %,$(HTMLDIR)/%,$(HTMLFILES)) $(patsubst %,$(MDDIR)/source/%,$(MDFILES)) ifneq ($(strip $(MINGW)),) $(ARCH_BINDIR)/$(START_UP): grass.sh diff --git a/man/Makefile b/man/Makefile index 1a189c8d7c8..4ad90b29a74 100644 --- a/man/Makefile +++ b/man/Makefile @@ -12,7 +12,11 @@ DSTFILES := \ $(HTMLDIR)/grass_icon.png \ $(HTMLDIR)/jquery.fixedheadertable.min.js \ $(HTMLDIR)/parser_standard_options.css \ - $(HTMLDIR)/parser_standard_options.js + $(HTMLDIR)/parser_standard_options.js \ + $(MDDIR)/mkdocs.yml \ + $(MDDIR)/source/grass_logo.png \ + $(MDDIR)/source/grassdocs.css \ + $(MDDIR)/overrides/partials/footer.html categories = \ d:display \ @@ -34,7 +38,11 @@ IDXSRC = full_index index topics keywords graphical_index manual_gallery class_g INDICES := $(patsubst %,$(HTMLDIR)/%.html,$(IDXSRC)) +IDXSRC_MD = full_index index topics keywords graphical_index manual_gallery parser_standard_options $(IDXCATS) +INDICES_MD := $(patsubst %,$(MDDIR)/source/%.md,$(IDXSRC_MD)) + ALL_HTML := $(wildcard $(HTMLDIR)/*.*.html) +ALL_MD := $(wildcard $(MDDIR)/source/*.*.html) ifneq (@(type sphinx-build2 > /dev/null),) SPHINXBUILD = sphinx-build2 @@ -44,10 +52,12 @@ SPHINXBUILD = sphinx-build endif default: $(DSTFILES) - @echo "Generating HTML manual pages index (help system)..." + @echo "Generating manual pages index (help system)..." $(MAKE) $(INDICES) $(call build,check) $(MAKE) manpages + $(MAKE) $(INDICES_MD) +# $(MAKE) build-mkdocs # This must be a separate target so that evaluation of $(MANPAGES) # is delayed until the indices have been generated @@ -60,32 +70,37 @@ manpages: define build GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \ + MDDIR="${MDDIR}" \ VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \ $(PYTHON) ./build_$(1).py $(2) endef define build_topics GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \ + MDDIR="${MDDIR}" \ VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \ - $(PYTHON) ./build_topics.py $(HTMLDIR) + $(PYTHON) ./build_topics.py endef define build_keywords GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \ + MDDIR="${MDDIR}" \ VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \ - $(PYTHON) ./build_keywords.py $(HTMLDIR) + $(PYTHON) ./build_keywords.py endef define build_graphical_index GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \ + MDDIR="${MDDIR}" \ VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \ - $(PYTHON) ./build_graphical_index.py $(HTMLDIR) + $(PYTHON) ./build_graphical_index.py endef define build_manual_gallery GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \ + MDDIR="${MDDIR}" \ VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \ - $(PYTHON) ./build_manual_gallery.py $(HTMLDIR) + $(PYTHON) ./build_manual_gallery.py endef define build_pso @@ -95,8 +110,18 @@ GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \ -f "grass" -o "$(HTMLDIR)/parser_standard_options.html" -p 'id="opts_table" class="scroolTable"' endef +define build_pso_md +GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \ + MDDIR="${MDDIR}" \ + VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \ + $(PYTHON) ./parser_standard_options.py -t "$(GRASS_HOME)/lib/gis/parser_standard_options.c" \ + -f "grass" -o "$(MDDIR)/source/parser_standard_options.md" +endef + $(HTMLDIR)/topics.html: $(ALL_HTML) +$(MDDIR)/source/topics.md: $(ALL_MD) + define build_class_graphical GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \ VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \ @@ -107,30 +132,56 @@ $(HTMLDIR)/topics.html: $(ALL_HTML) build_topics.py $(call build_topics) touch $@ +$(MDDIR)/source/topics.md: $(ALL_MD) build_topics.py + $(call build_topics) + touch $@ + $(HTMLDIR)/full_index.html: $(ALL_HTML) build_full_index.py build_html.py $(call build,full_index) touch $@ +$(MDDIR)/source/full_index.md: $(ALL_MD) build_full_index.py build_html.py + $(call build,full_index) + touch $@ + $(HTMLDIR)/index.html: build_index.py build_html.py $(call build,index) touch $@ +$(MDDIR)/source/index.md: build_index.py build_md.py + $(call build,index) + touch $@ + $(HTMLDIR)/keywords.html: $(ALL_HTML) $(call build_keywords) touch $@ +$(MDDIR)/source/keywords.md: $(ALL_MD) + $(call build_keywords) + touch $@ $(HTMLDIR)/graphical_index.html: $(ALL_HTML) $(call build_graphical_index) touch $@ +$(MDDIR)/source/graphical_index.md: $(ALL_MD) + $(call build_graphical_index) + touch $@ + $(HTMLDIR)/manual_gallery.html: $(ALL_HTML) $(call build_manual_gallery) +$(MDDIR)/source/manual_gallery.md: $(ALL_MD) + $(call build_manual_gallery) + $(HTMLDIR)/parser_standard_options.html: $(ALL_HTML) $(call build_pso) touch $@ +$(MDDIR)/source/parser_standard_options.md: $(ALL_MD) + $(call build_pso_md) + touch $@ + # TODO: this should be done in the same way as category_rule $(HTMLDIR)/class_graphical.html: $(ALL_HTML) $(call build_class_graphical) @@ -144,12 +195,24 @@ endef $(foreach cat,$(categories),$(eval $(call category_rule,$(firstword $(subst :, ,$(cat))),$(lastword $(subst :, ,$(cat)))))) +define category_rule_md +$$(MDDIR)/source/$(2).md: $$(wildcard $$(MDDIR)/source/$(1).*.md) build_class.py build_md.py + $$(call build,class,$(1) $(2)) + touch $$@ +endef + +$(foreach cat,$(categories),$(eval $(call category_rule_md,$(firstword $(subst :, ,$(cat))),$(lastword $(subst :, ,$(cat)))))) + + $(HTMLDIR)/grassdocs.css: grassdocs.css $(INSTALL_DATA) $< $@ $(HTMLDIR)/grass_logo.png: grass_logo.png $(INSTALL_DATA) $< $@ +$(MDDIR)/source/grass_logo.png: grass_logo.png + $(INSTALL_DATA) $< $@ + $(HTMLDIR)/hamburger_menu.svg: hamburger_menu.svg $(INSTALL_DATA) $< $@ @@ -167,3 +230,20 @@ $(HTMLDIR)/parser_standard_options.js: parser_standard_options.js $(HTMLDIR)/parser_standard_options.css: parser_standard_options.css $(INSTALL_DATA) $< $@ + +$(MDDIR)/mkdocs.yml: mkdocs/mkdocs.yml + $(INSTALL_DATA) $< $@ + +$(MDDIR)/source/grassdocs.css: mkdocs/grassdocs.css + $(INSTALL_DATA) $< $@ + +$(MDDIR)/overrides/partials/footer.html: mkdocs/overrides/partials/footer.html | $(MDDIR)/overrides/partials + $(INSTALL_DATA) $< $@ + +$(MDDIR)/overrides/partials: + $(MKDIR) $@ + +build-mkdocs: + @cd $(MDDIR) ; SITE_NAME="GRASS GIS $(GRASS_VERSION_NUMBER) Reference Manual" \ + COPYRIGHT="© 2003-$(GRASS_VERSION_DATE) GRASS Development Team, GRASS GIS $(GRASS_VERSION_NUMBER) Reference Manual" \ + mkdocs build diff --git a/man/build.py b/man/build.py new file mode 100644 index 00000000000..9c394a8749d --- /dev/null +++ b/man/build.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 + +# utilities for generating HTML indices +# (C) 2003-2024 Markus Neteler and the GRASS Development Team +# Authors: +# Markus Neteler +# Glynn Clements +# Luca Delucchi + +import os +import string +from datetime import datetime +from pathlib import Path + +# TODO: better fix this in include/Make/Html.make, see bug RT #5361 + +# exclude following list of modules from help index: + +exclude_mods = [ + "i.find", + "r.watershed.ram", + "r.watershed.seg", + "v.topo.check", + "helptext.html", +] + +# these modules don't use G_parser() + +desc_override = { + "g.parser": "Provides automated parser, GUI, and help support for GRASS scipts.", + "r.li.daemon": "Support module for r.li landscape index calculations.", +} + +# File template pieces follow + +message_tmpl = string.Template( + r"""Generated HTML docs in ${man_dir}/index.html +---------------------------------------------------------------------- +Following modules are missing the 'modulename.html' file in src code: +""" +) + +############################################################################ + + +def check_for_desc_override(basename): + return desc_override.get(basename) + + +def read_file(name): + return Path(name).read_text() + + +def write_file(name, contents): + Path(name).write_text(contents) + + +def try_mkdir(path): + try: + os.mkdir(path) + except OSError: + pass + + +def replace_file(name): + temp = name + ".tmp" + if ( + os.path.exists(name) + and os.path.exists(temp) + and read_file(name) == read_file(temp) + ): + os.remove(temp) + else: + try: + os.remove(name) + except OSError: + pass + os.rename(temp, name) + + +def copy_file(src, dst): + write_file(dst, read_file(src)) + + +def get_files(man_dir, cls=None, ignore_gui=True, extension="html"): + for cmd in sorted(os.listdir(man_dir)): + if ( + cmd.endswith(f".{extension}") + and (cls in (None, "*") or cmd.startswith(cls + ".")) + and (cls != "*" or len(cmd.split(".")) >= 3) + and cmd not in [f"full_index.{extension}", f"index.{extension}"] + and cmd not in exclude_mods + and ((ignore_gui and not cmd.startswith("wxGUI.")) or not ignore_gui) + ): + yield cmd + + +def write_header(f, title, ismain=False, body_width="99%", template="html"): + if template == "html": + from build_html import header1_tmpl, macosx_tmpl, header2_tmpl + else: + from build_md import header1_tmpl, macosx_tmpl, header2_tmpl + f.write(header1_tmpl.substitute(title=title)) + if ismain and macosx: + f.write( + macosx_tmpl.substitute(grass_version=grass_version, grass_mmver=grass_mmver) + ) + f.write(header2_tmpl.substitute(grass_version=grass_version, body_width=body_width)) + + +def write_cmd_overview(f, template="html"): + if template == "html": + from build_html import overview_tmpl + else: + from build_md import overview_tmpl + f.write( + overview_tmpl.substitute( + grass_version_major=grass_version_major, + grass_version_minor=grass_version_minor, + ) + ) + + +def write_footer(f, index_url, year=None, template="html"): + if template == "html": + from build_html import footer_tmpl + else: + from build_md import footer_tmpl + cur_year = default_year if year is None else year + f.write( + footer_tmpl.substitute( + grass_version=grass_version, index_url=index_url, year=cur_year + ) + ) + + +def to_title(name): + """Convert name of command class/family to form suitable for title""" + if name == "PostScript": + return name + return name.capitalize() + + +############################################################################ + +arch_dist_dir = os.environ["ARCH_DISTDIR"] +gisbase = os.environ["GISBASE"] +grass_version = os.getenv("VERSION_NUMBER", "unknown") +grass_version_major = grass_version.split(".")[0] +grass_version_minor = grass_version.split(".")[1] +grass_mmver = ".".join(grass_version.split(".")[0:2]) +macosx = "darwin" in os.environ["ARCH"].lower() +default_year = os.getenv("VERSION_DATE") +if not default_year: + default_year = str(datetime.now().year) + +############################################################################ diff --git a/man/build_check.py b/man/build_check.py index 9d9675fb7ac..1dc578c56af 100644 --- a/man/build_check.py +++ b/man/build_check.py @@ -9,13 +9,14 @@ import sys import os -from build_html import html_dir, message_tmpl, html_files, read_file +from build import message_tmpl, get_files, read_file +from build_html import man_dir -os.chdir(html_dir) +os.chdir(man_dir) -sys.stdout.write(message_tmpl.substitute(html_dir=html_dir)) +sys.stdout.write(message_tmpl.substitute(man_dir=man_dir)) -for cmd in html_files("*"): +for cmd in get_files(man_dir, "*"): if "DESCRIPTION" not in read_file(cmd): sys.stdout.write("%s\n" % cmd[:-5]) diff --git a/man/build_class.py b/man/build_class.py index 564fb5c20e6..a75fe62fe58 100644 --- a/man/build_class.py +++ b/man/build_class.py @@ -9,67 +9,86 @@ import sys import os -from build_html import ( - html_dir, - write_html_header, - grass_version, - modclass_intro_tmpl, - modclass_tmpl, - to_title, - html_files, - check_for_desc_override, - get_desc, - desc2_tmpl, - write_html_footer, - replace_file, -) - - no_intro_page_classes = ["display", "general", "miscellaneous", "postscript"] -os.chdir(html_dir) - -# write separate module pages: - -# for all module groups: -cls = sys.argv[1] -modclass = sys.argv[2] -year = None -if len(sys.argv) > 3: - year = sys.argv[3] - -filename = modclass + ".html" - -f = open(filename + ".tmp", "w") - -write_html_header( - f, - "%s modules - GRASS GIS %s Reference Manual" - % (modclass.capitalize(), grass_version), -) -modclass_lower = modclass.lower() -modclass_visible = modclass -if modclass_lower not in no_intro_page_classes: - if modclass_visible == "raster3d": - # covert keyword to nice form - modclass_visible = "3D raster" - f.write( - modclass_intro_tmpl.substitute( - modclass=modclass_visible, modclass_lower=modclass_lower + +def build_class(ext): + if ext == "html": + from build_html import ( + modclass_tmpl, + get_desc, + desc2_tmpl, + modclass_intro_tmpl, + man_dir, + ) + else: + from build_md import ( + modclass_tmpl, + get_desc, + desc2_tmpl, + modclass_intro_tmpl, + man_dir, ) + + os.chdir(man_dir) + + filename = modclass + f".{ext}" + f = open(filename + ".tmp", "w") + + write_header( + f, + "{} modules - GRASS GIS {} Reference Manual".format( + modclass.capitalize(), grass_version + ), + template=ext, ) -f.write(modclass_tmpl.substitute(modclass=to_title(modclass_visible))) + modclass_lower = modclass.lower() + modclass_visible = modclass + if modclass_lower not in no_intro_page_classes: + if modclass_visible == "raster3d": + # convert keyword to nice form + modclass_visible = "3D raster" + f.write( + modclass_intro_tmpl.substitute( + modclass=modclass_visible, modclass_lower=modclass_lower + ) + ) + f.write(modclass_tmpl.substitute(modclass=to_title(modclass_visible))) + + # for all modules: + for cmd in get_files(man_dir, cls, extension=ext): + basename = os.path.splitext(cmd)[0] + desc = check_for_desc_override(basename) + if desc is None: + desc = get_desc(cmd) + f.write(desc2_tmpl.substitute(cmd=cmd, basename=basename, desc=desc)) + if ext == "html": + f.write("\n") + + write_footer(f, f"index.{ext}", year, template=ext) + + f.close() + replace_file(filename) -# for all modules: -for cmd in html_files(cls): - basename = os.path.splitext(cmd)[0] - desc = check_for_desc_override(basename) - if desc is None: - desc = get_desc(cmd) - f.write(desc2_tmpl.substitute(cmd=cmd, basename=basename, desc=desc)) -f.write("\n") -write_html_footer(f, "index.html", year) +if __name__ == "__main__": + # for all module groups: + cls = sys.argv[1] + modclass = sys.argv[2] + year = None + if len(sys.argv) > 3: + year = sys.argv[3] + + from build import ( + grass_version, + to_title, + check_for_desc_override, + replace_file, + get_files, + write_header, + write_footer, + ) + + build_class("html") -f.close() -replace_file(filename) + build_class("md") diff --git a/man/build_class_graphical.py b/man/build_class_graphical.py index 83338f1942c..b1bd6eede7f 100644 --- a/man/build_class_graphical.py +++ b/man/build_class_graphical.py @@ -17,20 +17,23 @@ import os import fnmatch -# from build_html import * -from build_html import ( +from build import ( default_year, - header1_tmpl, grass_version, - modclass_intro_tmpl, to_title, - html_files, + get_files, check_for_desc_override, - get_desc, - write_html_footer, + write_footer, replace_file, ) +from build_html import ( + header1_tmpl, + modclass_intro_tmpl, + get_desc, + man_dir, +) + header_graphical_index_tmpl = """\ @@ -156,7 +159,7 @@ def generate_page_for_category( output.write('
    ') # for all modules: - for cmd in html_files(short_family, ignore_gui=False): + for cmd in get_files(man_dir, short_family, ignore_gui=False): basename = os.path.splitext(cmd)[0] desc = check_for_desc_override(basename) if desc is None: @@ -184,7 +187,7 @@ def generate_page_for_category( output.write("
") - write_html_footer(output, "index.html", year) + write_footer(output, "index.html", year, template="html") output.close() replace_file(filename) diff --git a/man/build_full_index.py b/man/build_full_index.py index 7d2ce03e342..150f28fd00b 100644 --- a/man/build_full_index.py +++ b/man/build_full_index.py @@ -11,77 +11,100 @@ from operator import itemgetter -from build_html import ( - html_dir, - grass_version, - html_files, - write_html_header, - write_html_footer, - check_for_desc_override, - get_desc, - replace_file, - to_title, - full_index_header, - toc, - cmd2_tmpl, - desc1_tmpl, -) - -year = None -if len(sys.argv) > 1: - year = sys.argv[1] - -os.chdir(html_dir) - -# TODO: create some master function/dict somewhere -class_labels = { - "d": "display", - "db": "database", - "g": "general", - "i": "imagery", - "m": "miscellaneous", - "ps": "PostScript", - "r": "raster", - "r3": "3D raster", - "t": "temporal", - "v": "vector", -} - -classes = [] -for cmd in html_files("*"): - prefix = cmd.split(".")[0] - if prefix not in [item[0] for item in classes]: - classes.append((prefix, class_labels.get(prefix, prefix))) -classes.sort(key=itemgetter(0)) - -# begin full index: -filename = "full_index.html" -f = open(filename + ".tmp", "w") - -write_html_header( - f, "GRASS GIS %s Reference Manual: Full index" % grass_version, body_width="80%" -) - -# generate main index of all modules: -f.write(full_index_header) - -f.write(toc) - -# for all module groups: -for cls, cls_label in classes: - f.write(cmd2_tmpl.substitute(cmd_label=to_title(cls_label), cmd=cls)) - # for all modules: - for cmd in html_files(cls): - basename = os.path.splitext(cmd)[0] - desc = check_for_desc_override(basename) - if desc is None: - desc = get_desc(cmd) - f.write(desc1_tmpl.substitute(cmd=cmd, basename=basename, desc=desc)) - f.write("\n") - -write_html_footer(f, "index.html", year) - -f.close() -replace_file(filename) - -# done full index + +def build_full_index(ext): + if ext == "html": + from build_html import ( + man_dir, + full_index_header, + cmd2_tmpl, + desc1_tmpl, + get_desc, + toc, + ) + else: + from build_md import ( + man_dir, + full_index_header, + cmd2_tmpl, + desc1_tmpl, + get_desc, + ) + + os.chdir(man_dir) + + # TODO: create some master function/dict somewhere + class_labels = { + "d": "display", + "db": "database", + "g": "general", + "i": "imagery", + "m": "miscellaneous", + "ps": "PostScript", + "r": "raster", + "r3": "3D raster", + "t": "temporal", + "v": "vector", + } + + classes = [] + for cmd in get_files(man_dir, "*", extension=ext): + prefix = cmd.split(".")[0] + if prefix not in [item[0] for item in classes]: + classes.append((prefix, class_labels.get(prefix, prefix))) + classes.sort(key=itemgetter(0)) + + # begin full index: + filename = f"full_index.{ext}" + f = open(filename + ".tmp", "w") + + write_header( + f, + "GRASS GIS {} Reference Manual - Full index".format(grass_version), + body_width="80%", + template=ext, + ) + + # generate main index of all modules: + f.write(full_index_header) + + if ext == "html": + f.write(toc) + + # for all module groups: + for cls, cls_label in classes: + f.write(cmd2_tmpl.substitute(cmd_label=to_title(cls_label), cmd=cls)) + # for all modules: + for cmd in get_files(man_dir, cls, extension=ext): + basename = os.path.splitext(cmd)[0] + desc = check_for_desc_override(basename) + if desc is None: + desc = get_desc(cmd) + f.write(desc1_tmpl.substitute(cmd=cmd, basename=basename, desc=desc)) + if ext == "html": + f.write("\n") + + write_footer(f, f"index.{ext}", year, template=ext) + + f.close() + replace_file(filename) + + +if __name__ == "__main__": + year = None + if len(sys.argv) > 1: + year = sys.argv[1] + + from build import ( + get_files, + write_footer, + write_header, + to_title, + grass_version, + check_for_desc_override, + replace_file, + ) + + build_full_index("html") + + build_full_index("md") diff --git a/man/build_graphical_index.py b/man/build_graphical_index.py index 04e981c021e..d46574af0a8 100755 --- a/man/build_graphical_index.py +++ b/man/build_graphical_index.py @@ -14,73 +14,11 @@ ############################################################################# import os -import sys -from build_html import write_html_footer, grass_version, header1_tmpl - - -output_name = "graphical_index.html" +output_name = "graphical_index" year = os.getenv("VERSION_DATE") -# other similar strings are in a different file -# TODO: all HTML manual building needs refactoring (perhaps grass.tools?) -header_graphical_index_tmpl = """\ - - - - -
- -GRASS logo -
-

Graphical index of GRASS GIS modules

-""" - def std_img_name(name): return "gi_{0}.jpg".format(name) @@ -114,30 +52,57 @@ def std_img_name(name): ] -def main(): - html_dir = sys.argv[1] +def main(ext): + if ext == "html": + from build_html import ( + header1_tmpl, + header_graphical_index_tmpl, + man_dir, + ) + else: + from build_md import ( + header1_tmpl, + header_graphical_index_tmpl, + man_dir, + ) - with open(os.path.join(html_dir, output_name), "w") as output: + with open(os.path.join(man_dir, output_name + f".{ext}"), "w") as output: output.write( header1_tmpl.substitute( - title="GRASS GIS %s Reference " - "Manual: Graphical index" % grass_version + title=f"GRASS GIS {grass_version} Reference Manual - Graphical index" ) ) output.write(header_graphical_index_tmpl) - output.write('
    \n') + if ext == "html": + output.write('
      \n') for html_file, image, label in index_items: - output.write( - "
    • " - '' - '' - '{name}' - "" - "
      • \n".format(html=html_file, img=image, name=label) - ) - output.write("
    ") - write_html_footer(output, "index.html", year) + if ext == "html": + output.write( + "
  • " + '' + '' + '{name}' + "" + "
    • \n".format(html=html_file, img=image, name=label) + ) + else: + output.write( + "- [![{name}]({img})]({link})".format( + link=html_file, img=image, name=label + ) + ) + + if ext == "html": + output.write("
") + write_footer(output, f"index.{ext}", year, template=ext) if __name__ == "__main__": - main() + from build import ( + write_footer, + grass_version, + ) + + main("html") + + main("md") diff --git a/man/build_html.py b/man/build_html.py index a8cc4c057a5..db5a424c738 100644 --- a/man/build_html.py +++ b/man/build_html.py @@ -1,34 +1,5 @@ -#!/usr/bin/env python3 - -# utilities for generating HTML indices -# (C) 2003-2024 Markus Neteler and the GRASS Development Team -# Authors: -# Markus Neteler -# Glynn Clements -# Luca Delucchi - import os import string -from datetime import datetime - -# TODO: better fix this in include/Make/Html.make, see bug RT #5361 - -# exclude following list of modules from help index: - -exclude_mods = [ - "i.find", - "r.watershed.ram", - "r.watershed.seg", - "v.topo.check", - "helptext.html", -] - -# these modules don't use G_parser() - -desc_override = { - "g.parser": "Provides automated parser, GUI, and help support for GRASS scipts.", - "r.li.daemon": "Support module for r.li landscape index calculations.", -} # File template pieces follow @@ -310,15 +281,6 @@ """ # " - -message_tmpl = string.Template( - r"""Generated HTML docs in ${html_dir}/index.html ----------------------------------------------------------------------- -Following modules are missing the 'modulename.html' file in src code: -""" -) -# " - moduletopics_tmpl = string.Template( r"""
  • ${name}
    • @@ -385,91 +347,62 @@ """ # " -############################################################################ - - -def check_for_desc_override(basename): - return desc_override.get(basename) - - -def read_file(name): - f = open(name) - s = f.read() - f.close() - return s - - -def write_file(name, contents): - f = open(name, "w") - f.write(contents) - f.close() - - -def try_mkdir(path): - try: - os.mkdir(path) - except OSError: - pass - - -def replace_file(name): - temp = name + ".tmp" - if ( - os.path.exists(name) - and os.path.exists(temp) - and read_file(name) == read_file(temp) - ): - os.remove(temp) - else: - try: - os.remove(name) - except OSError: - pass - os.rename(temp, name) - - -def copy_file(src, dst): - write_file(dst, read_file(src)) - +# TODO: all HTML manual building needs refactoring (perhaps grass.tools?) +header_graphical_index_tmpl = """\ + + + + +
    -def write_html_footer(f, index_url, year=None): - cur_year = default_year if year is None else year - f.write( - footer_tmpl.substitute( - grass_version=grass_version, index_url=index_url, year=cur_year - ) - ) +GRASS logo +
    +

    Graphical index of GRASS GIS modules

    +""" def get_desc(cmd): @@ -496,25 +429,8 @@ def get_desc(cmd): return "" -def to_title(name): - """Convert name of command class/family to form suitable for title""" - if name == "PostScript": - return name - return name.capitalize() - - ############################################################################ -arch_dist_dir = os.environ["ARCH_DISTDIR"] -html_dir = os.path.join(arch_dist_dir, "docs", "html") -gisbase = os.environ["GISBASE"] -grass_version = os.getenv("VERSION_NUMBER", "unknown") -grass_version_major = grass_version.split(".")[0] -grass_version_minor = grass_version.split(".")[1] -grass_mmver = ".".join(grass_version.split(".")[0:2]) -macosx = "darwin" in os.environ["ARCH"].lower() -default_year = os.getenv("VERSION_DATE") -if not default_year: - default_year = str(datetime.now().year) +man_dir = os.path.join(os.environ["ARCH_DISTDIR"], "docs", "html") ############################################################################ diff --git a/man/build_index.py b/man/build_index.py index 12de7f01162..ef4dbc0dcc1 100644 --- a/man/build_index.py +++ b/man/build_index.py @@ -9,26 +9,42 @@ import sys import os -from build_html import ( - html_dir, - grass_version, - write_html_header, - write_html_cmd_overview, - write_html_footer, +from build import ( + write_header, + write_cmd_overview, + write_footer, replace_file, + grass_version, ) -os.chdir(html_dir) - -filename = "index.html" -f = open(filename + ".tmp", "w") - year = None if len(sys.argv) > 1: year = sys.argv[1] -write_html_header(f, "GRASS GIS %s Reference Manual" % grass_version, True) -write_html_cmd_overview(f) -write_html_footer(f, "index.html", year) -f.close() -replace_file(filename) + +def build_index(ext): + if ext == "html": + from build_html import ( + man_dir, + ) + else: + from build_md import ( + man_dir, + ) + + filename = f"index.{ext}" + os.chdir(man_dir) + with open(filename + ".tmp", "w") as f: + write_header( + f, f"GRASS GIS {grass_version} Reference Manual", True, template=ext + ) + write_cmd_overview(f) + write_footer(f, f"index.{ext}", year, template=ext) + replace_file(filename) + + +if __name__ == "__main__": + + build_index("html") + + build_index("md") diff --git a/man/build_keywords.py b/man/build_keywords.py index b0dfe95e0e3..5793bb9b4d3 100644 --- a/man/build_keywords.py +++ b/man/build_keywords.py @@ -21,13 +21,6 @@ import os import sys import glob -from build_html import ( - grass_version, - header1_tmpl, - headerkeywords_tmpl, - write_html_footer, -) - blacklist = [ "Display", @@ -42,34 +35,25 @@ "Vector", ] -path = sys.argv[1] addons_path = None -if len(sys.argv) >= 3: - addons_path = sys.argv[2] +if len(sys.argv) >= 2: + addons_path = sys.argv[1] year = os.getenv("VERSION_DATE") -keywords = {} - -htmlfiles = glob.glob(os.path.join(path, "*.html")) -if addons_path: - addons_man_files = glob.glob(os.path.join(addons_path, "*.html")) - htmlfiles.extend(addons_man_files) - -char_list = {} - -def get_module_man_html_file_path(module): +def get_module_man_file_path(man_dir, module, addons_man_files): """Get module manual HTML file path :param str module: module manual HTML file name e.g. v.surf.rst.html + :param addons_man_files: list of HTML manual files :return str module_path: core/addon module manual HTML file path """ if addons_path and module in ",".join(addons_man_files): module_path = os.path.join(addons_path, module) module_path = module_path.replace( - os.path.commonpath([path, module_path]), + os.path.commonpath([man_dir, module_path]), ".", ) else: @@ -77,97 +61,142 @@ def get_module_man_html_file_path(module): return module_path -for html_file in htmlfiles: - fname = os.path.basename(html_file) - with open(html_file) as f: - lines = f.readlines() - # TODO maybe move to Python re (regex) - # remove empty lines - lines = [x for x in lines if x != "\n"] - try: - index_keys = lines.index("

    KEYWORDS

    \n") + 1 - index_desc = lines.index("

    NAME

    \n") + 1 - except Exception: - continue - try: - keys = lines[index_keys].split(",") - except Exception: - continue - for key in keys: - key = key.strip() - try: - key = key.split(">")[1].split("<")[0] - except Exception: - pass - if not key: - sys.exit("Empty keyword from file %s line: %s" % (fname, lines[index_keys])) - if key not in keywords.keys(): - keywords[key] = [] - keywords[key].append(fname) - elif fname not in keywords[key]: - keywords[key].append(fname) - -for black in blacklist: - try: - del keywords[black] - except Exception: +def build_keywords(ext): + if ext == "html": + from build_html import header1_tmpl, headerkeywords_tmpl, man_dir + else: + from build_md import ( + header1_tmpl, + headerkeywords_tmpl, + man_dir, + ) + + keywords = {} + + files = glob.glob(os.path.join(man_dir, f"*.{ext}")) + # TODO: add markdown support + if addons_path: + addons_man_files = glob.glob(os.path.join(addons_path, f"*.{ext}")) + files.extend(addons_man_files) + else: + addons_man_files = [] + + char_list = {} + + for in_file in files: + fname = os.path.basename(in_file) + with open(in_file) as f: + lines = f.readlines() + + if ext == "html": + # TODO maybe move to Python re (regex) + try: + index_keys = lines.index("

    KEYWORDS

    \n") + 1 + except Exception: + continue + try: + keys = [] + for k in lines[index_keys].split(","): + keys.append(k.strip().split(">")[1].split("<")[0]) + except Exception: + continue + else: + keys = [] + for line in lines: + if "keywords:" in line: + keys = [x.strip() for x in line.split(":", 1)[1].strip().split(",")] + break + + for key in keys: + if key not in keywords.keys(): + keywords[key] = [] + keywords[key].append(fname) + elif fname not in keywords[key]: + keywords[key].append(fname) + + for black in blacklist: try: - del keywords[black.lower()] + del keywords[black] except Exception: - continue - -for key in sorted(keywords.keys()): - # this list it is useful to create the TOC using only the first - # character for keyword - firstchar = key[0].lower() - if firstchar not in char_list.keys(): - char_list[str(firstchar)] = key - elif firstchar in char_list.keys(): - if key.lower() < char_list[str(firstchar)].lower(): - char_list[str(firstchar.lower())] = key - -keywordsfile = open(os.path.join(path, "keywords.html"), "w") -keywordsfile.write( - header1_tmpl.substitute( - title="GRASS GIS %s Reference Manual: Keywords index" % grass_version + try: + del keywords[black.lower()] + except Exception: + continue + + for key in sorted(keywords.keys()): + # this list it is useful to create the TOC using only the first + # character for keyword + firstchar = key[0].lower() + if firstchar not in char_list.keys(): + char_list[str(firstchar)] = key + elif firstchar in char_list.keys(): + if key.lower() < char_list[str(firstchar)].lower(): + char_list[str(firstchar.lower())] = key + + keywordsfile = open(os.path.join(man_dir, f"keywords.{ext}"), "w") + keywordsfile.write( + header1_tmpl.substitute( + title=f"GRASS GIS {grass_version} Reference Manual - Keywords index" + ) ) -) -keywordsfile.write(headerkeywords_tmpl) -keywordsfile.write("
    ") - -sortedKeys = sorted(keywords.keys(), key=lambda s: s.lower()) - -for key in sortedKeys: - keyword_line = '
    %s
    ' % ( - key, - key, + keywordsfile.write(headerkeywords_tmpl) + if ext == "html": + keywordsfile.write("
    ") + sortedKeys = sorted(keywords.keys(), key=lambda s: s.lower()) + + for key in sortedKeys: + if ext == "html": + keyword_line = ( + '
    {key}
    '.format( + key=key + ) + ) + else: + keyword_line = f"### **{key}**\n" + for value in sorted(keywords[key]): + man_file_path = get_module_man_file_path(man_dir, value, addons_man_files) + if ext == "html": + keyword_line += ( + f' {value.replace(f".{ext}", "")},' + ) + else: + keyword_line += f' [{value.rsplit(".", 1)[0]}]({man_file_path}),' + keyword_line = keyword_line.rstrip(",") + if ext == "html": + keyword_line += "
    " + keyword_line += "\n" + keywordsfile.write(keyword_line) + if ext == "html": + keywordsfile.write("
    \n") + if ext == "html": + # create toc + toc = '
    \n

    Table of contents

    ' + test_length = 0 + all_keys = len(char_list.keys()) + for k in sorted(char_list.keys()): + test_length += 1 + # toc += '

  • %s
    • ' % (char_list[k], k) + if test_length % 4 == 0 and test_length != all_keys: + toc += '\n%s, ' % (char_list[k], k) + elif test_length % 4 == 0 and test_length == all_keys: + toc += '\n%s' % (char_list[k], k) + elif test_length == all_keys: + toc += '%s' % (char_list[k], k) + else: + toc += '%s, ' % (char_list[k], k) + toc += "

    \n" + keywordsfile.write(toc) + + write_footer(keywordsfile, f"index.{ext}", year, template=ext) + keywordsfile.close() + + +if __name__ == "__main__": + from build import ( + grass_version, + write_footer, ) - for value in sorted(keywords[key]): - keyword_line += ( - f' ' - f'{value.replace(".html", "")},' - ) - keyword_line = keyword_line.rstrip(",") - keyword_line += "
    \n" - keywordsfile.write(keyword_line) -keywordsfile.write("
    \n") -# create toc -toc = '
    \n

    Table of contents

    ' -test_length = 0 -all_keys = len(char_list.keys()) -for k in sorted(char_list.keys()): - test_length += 1 - # toc += '

  • %s
    • ' % (char_list[k], k) - if test_length % 4 == 0 and test_length != all_keys: - toc += '\n%s, ' % (char_list[k], k) - elif test_length % 4 == 0 and test_length == all_keys: - toc += '\n%s' % (char_list[k], k) - elif test_length == all_keys: - toc += '%s' % (char_list[k], k) - else: - toc += '%s, ' % (char_list[k], k) -toc += "

    \n" -keywordsfile.write(toc) -write_html_footer(keywordsfile, "index.html", year) -keywordsfile.close() + build_keywords("html") + + build_keywords("md") diff --git a/man/build_manual_gallery.py b/man/build_manual_gallery.py index a8b077be8f9..2fa6ace609d 100755 --- a/man/build_manual_gallery.py +++ b/man/build_manual_gallery.py @@ -15,15 +15,9 @@ import os from pathlib import Path -import sys import fnmatch import re -from build_html import write_html_footer, grass_version, header1_tmpl - - -output_name = "manual_gallery.html" - img_extensions = ["png", "jpg", "gif"] img_patterns = ["*." + extension for extension in img_extensions] @@ -94,10 +88,14 @@ """ -def img_in_html(filename, imagename) -> bool: +def img_in_file(filename, imagename, ext) -> bool: # for some reason, calling search just once is much faster # than calling it on every line (time is spent in _compile) - pattern = re.compile("".format(imagename)) + if ext == "html": + pattern = re.compile("".format(imagename)) + else: + # expecting markdown + pattern = re.compile(r"!\[(.*?)\]\({0}\)".format(imagename)) return bool(re.search(pattern, Path(filename).read_text())) @@ -135,51 +133,82 @@ def title_from_names(module_name, img_name): return "{name}".format(name=module_name) -def get_module_name(filename): - return filename.replace(".html", "") +def get_module_name(filename, ext): + return filename.replace(f".{ext}", "") -def main(): - html_dir = sys.argv[1] +def main(ext): + if ext == "html": + from build_html import ( + header1_tmpl, + man_dir, + ) + else: + from build_md import ( + header1_tmpl, + man_dir, + ) + + output_name = f"manual_gallery.{ext}" - html_files = get_files( - html_dir, - ["*.html"], - exclude_patterns=[output_name, "*_graphical.html", "graphical_index.html"], + man_files = get_files( + man_dir, + [f"*.{ext}"], + exclude_patterns=[output_name, f"*_graphical.{ext}", f"graphical_index.{ext}"], ) - img_html_files = {} + img_files = {} - for filename in os.listdir(html_dir): + for filename in os.listdir(man_dir): if filename in img_blacklist: continue if file_matches(filename, img_patterns): - for html_file in html_files: - if img_in_html(os.path.join(html_dir, html_file), filename): - img_html_files[filename] = html_file - # for now suppose one image per html + for man_filename in man_files: + if img_in_file(os.path.join(man_dir, man_filename), filename, ext): + img_files[filename] = man_filename + # for now suppose one image per manual filename - with open(os.path.join(html_dir, output_name), "w") as output: + with open(os.path.join(man_dir, output_name), "w") as output: output.write( header1_tmpl.substitute( title="GRASS GIS %s Reference Manual: Manual gallery" % grass_version ) ) - output.write(header_graphical_index_tmpl) - output.write('
      \n') - for image, html_file in sorted(img_html_files.items()): - name = get_module_name(html_file) + if ext == "html": + output.write(header_graphical_index_tmpl) + output.write('
        \n') + for image, filename in sorted(img_files.items()): + name = get_module_name(filename, ext) title = title_from_names(name, image) - output.write( - "
      • " - '' - '' - '{name}' - "" - "
        • \n".format(html=html_file, img=image, title=title, name=name) - ) - output.write("
      ") - write_html_footer(output, "index.html", year) + if ext == "html": + output.write( + "
    • " + '' + '' + '{name}' + "" + "
      • \n".format(fn=filename, img=image, title=title, name=name) + ) + else: + output.write(f'[![{name}]({image} "{title}")]({filename})\n') + if ext == "html": + output.write("
    ") + write_footer(output, f"index.{ext}", year) + + return img_files if __name__ == "__main__": - main() + from build import ( + write_footer, + grass_version, + ) + + img_files_html = main("html") + + img_files_md = main("md") + + # TODO: img_files_html and img_files_md should be the same + # remove lines when fixed + for k in img_files_html: + if k not in img_files_md: + print(k) diff --git a/man/build_md.py b/man/build_md.py new file mode 100644 index 00000000000..4976c8f6364 --- /dev/null +++ b/man/build_md.py @@ -0,0 +1,266 @@ +import os +import string + +# File template pieces follow + +header1_tmpl = string.Template( + r"""--- +title: ${title} +author: GRASS Development Team +--- + +""" +) + +macosx_tmpl = string.Template( + r""" +AppleTitle: GRASS GIS ${grass_version} +AppleIcon: GRASS-${grass_mmver}/grass_icon.png +""" +) + +header2_tmpl = string.Template( + r"""# GRASS GIS ${grass_version} Reference Manual + +**Geographic Resources Analysis Support System**, commonly +referred to as [GRASS GIS](https://grass.osgeo.org), is a +[Geographic Information System](https://en.wikipedia.org/wiki/Geographic_information_system) +(GIS) used for geospatial data management and +analysis, image processing, graphics/maps production, spatial +modeling, and visualization. GRASS is currently used in academic and +commercial settings around the world, as well as by many governmental +agencies and environmental consulting companies. + +This reference manual details the use of modules distributed with +Geographic Resources Analysis Support System (GRASS), an open source +([GNU GPLed](https://www.gnu.org/licenses/gpl.html), image +processing and geographic information system (GIS). + +""" +) + +# TODO: avoid HTML tags +overview_tmpl = string.Template( + r""" + + + + + + + + + + + + + + + + + + + + + + + + +

     Quick Introduction

    + +

    +

    +

    +

    +

    +

    +

     Graphical User Interface

    + + + +

     Display

    + +

     General

    + +

     Addons

    + +

     Programmer's Manual

    + +

     Raster processing

    + +

     3D raster processing

    +

     Image processing

    +

     Vector processing

    +

     Database

    + +

     Temporal processing

    + +

     Cartography

    + +

     Miscellaneous & Variables

    + +

     Python

    + +
    +""" +) + +# footer_tmpl = string.Template( +# r""" +# ____ +# [Main index](${index_url}) | +# [Topics index](topics.md) | +# [Keywords index](keywords.md) | +# [Graphical index](graphical_index.md) | +# [Full index](full_index.md) + +# © 2003-${year} +# [GRASS Development Team](https://grass.osgeo.org), +# GRASS GIS ${grass_version} Reference Manual +# """ +# ) +# replaced by footer +footer_tmpl = string.Template("") + +cmd2_tmpl = string.Template( + r""" +### ${cmd_label} commands (${cmd}.*) + +| Module | Description | +|--------|-------------| +""" +) + +desc1_tmpl = string.Template( + r"""| [${basename}](${cmd}) | ${desc} | +""" +) + +modclass_intro_tmpl = string.Template( + r"""Go to [${modclass} introduction](${modclass_lower}intro.html) | [topics](topics.html) +""" +) +# " + + +modclass_tmpl = string.Template( + r"""Go [back to help overview](index.md) +### ${modclass} commands: +| Module | Description | +|--------|-------------| +""" +) + +desc2_tmpl = string.Template( + r"""| [${basename}](${cmd}) | ${desc} | +""" +) + +full_index_header = r"""Go [back to help overview](index.md) +""" + +moduletopics_tmpl = string.Template( + r""" +- [${name}](topic_${key}.html) +""" +) + +headertopics_tmpl = r"""# Topics +""" + +headerkeywords_tmpl = r"""# Keywords - Index of GRASS GIS modules +""" + +headerkey_tmpl = string.Template( + r"""# Topic: ${keyword} + +| Module | Description | +|--------|-------------| +""" +) + + +headerpso_tmpl = r""" +## Parser standard options +""" + +header_graphical_index_tmpl = """# Graphical index of GRASS GIS modules +""" + +############################################################################ + + +def get_desc(cmd): + desc = "" + with open(cmd) as f: + while True: + line = f.readline() + if not line: + return desc + if "description:" in line: + desc = line.split(":", 1)[1].strip() + break + + return desc + + +############################################################################ + +man_dir = os.path.join(os.environ["MDDIR"], "source") + +############################################################################ diff --git a/man/build_topics.py b/man/build_topics.py index be5f2962d2c..fa4d683f117 100644 --- a/man/build_topics.py +++ b/man/build_topics.py @@ -1,91 +1,138 @@ #!/usr/bin/env python3 # generates topics.html and topic_*.html -# (c) 2012 by the GRASS Development Team, Markus Neteler, Luca Delucchi +# (c) 2012-2024 by the GRASS Development Team import os -import sys import glob -from build_html import ( - grass_version, - header1_tmpl, - headertopics_tmpl, - headerkey_tmpl, - desc1_tmpl, - moduletopics_tmpl, - write_html_footer, -) - -path = sys.argv[1] + year = os.getenv("VERSION_DATE") min_num_modules_for_topic = 3 -keywords = {} - -htmlfiles = glob.glob1(path, "*.html") - -for fname in htmlfiles: - fil = open(os.path.join(path, fname)) - # TODO maybe move to Python re (regex) - lines = fil.readlines() - try: - index_keys = lines.index("

    KEYWORDS

    \n") + 1 - index_desc = lines.index("

    NAME

    \n") + 1 - except Exception: - continue - try: - key = lines[index_keys].split(",")[1].strip().replace(" ", "_") - key = key.split(">")[1].split("<")[0] - except Exception: - continue - try: - desc = lines[index_desc].split("-", 1)[1].strip() - except Exception: - desc.strip() - if key not in keywords.keys(): - keywords[key] = {} - keywords[key][fname] = desc - elif fname not in keywords[key]: - keywords[key][fname] = desc - -topicsfile = open(os.path.join(path, "topics.html"), "w") -topicsfile.write( - header1_tmpl.substitute( - title="GRASS GIS %s Reference Manual: Topics index" % grass_version - ) -) -topicsfile.write(headertopics_tmpl) -for key, values in sorted(keywords.items(), key=lambda s: s[0].lower()): - keyfile = open(os.path.join(path, "topic_%s.html" % key), "w") - keyfile.write( - header1_tmpl.substitute( - title="GRASS GIS " - "%s Reference Manual: Topic %s" % (grass_version, key.replace("_", " ")) +def build_topics(ext): + if ext == "html": + from build_html import ( + header1_tmpl, + headertopics_tmpl, + headerkey_tmpl, + desc1_tmpl, + moduletopics_tmpl, + man_dir, ) - ) - keyfile.write(headerkey_tmpl.substitute(keyword=key.replace("_", " "))) - num_modules = 0 - for mod, desc in sorted(values.items()): - num_modules += 1 - keyfile.write( - desc1_tmpl.substitute(cmd=mod, desc=desc, basename=mod.replace(".html", "")) + else: + from build_md import ( + header1_tmpl, + headertopics_tmpl, + headerkey_tmpl, + desc1_tmpl, + moduletopics_tmpl, + man_dir, ) - if num_modules >= min_num_modules_for_topic: - topicsfile.writelines( - [moduletopics_tmpl.substitute(key=key, name=key.replace("_", " "))] + + keywords = {} + + files = glob.glob1(man_dir, f"*.{ext}") + for fname in files: + fil = open(os.path.join(man_dir, fname)) + # TODO maybe move to Python re (regex) + lines = fil.readlines() + try: + if ext == "html": + index_keys = lines.index("

    KEYWORDS

    \n") + 1 + index_desc = lines.index("

    NAME

    \n") + 1 + else: + # expecting markdown + index_keys = lines.index("### KEYWORDS\n") + 3 + index_desc = lines.index("## NAME\n") + 2 + except Exception: + continue + try: + if ext == "html": + key = lines[index_keys].split(",")[1].strip().replace(" ", "_") + key = key.split(">")[1].split("<")[0] + else: + # expecting markdown + key = lines[index_keys].split("]")[0].lstrip("[") + except Exception: + continue + try: + desc = lines[index_desc].split("-", 1)[1].strip() + except Exception: + desc.strip() + + if key not in keywords.keys(): + keywords[key] = {} + keywords[key][fname] = desc + elif fname not in keywords[key]: + keywords[key][fname] = desc + + topicsfile = open(os.path.join(man_dir, f"topics.{ext}"), "w") + topicsfile.write( + header1_tmpl.substitute( + title="GRASS GIS %s Reference Manual - Topics index" % grass_version ) - keyfile.write("\n") - # link to the keywords index - # TODO: the labels in keywords index are with spaces and capitals - # this should be probably changed to lowercase with underscores - keyfile.write( - "

    See also the corresponding keyword" - ' {key}' - " for additional references.".format(key=key.replace("_", " ")) ) - write_html_footer(keyfile, "index.html", year) -topicsfile.write("\n") -write_html_footer(topicsfile, "index.html", year) -topicsfile.close() + topicsfile.write(headertopics_tmpl) + + for key, values in sorted(keywords.items(), key=lambda s: s[0].lower()): + keyfile = open(os.path.join(man_dir, f"topic_%s.{ext}" % key), "w") + if ext == "html": + keyfile.write( + header1_tmpl.substitute( + title="GRASS GIS " + "%s Reference Manual: Topic %s" + % (grass_version, key.replace("_", " ")) + ) + ) + keyfile.write(headerkey_tmpl.substitute(keyword=key.replace("_", " "))) + num_modules = 0 + for mod, desc in sorted(values.items()): + num_modules += 1 + keyfile.write( + desc1_tmpl.substitute( + cmd=mod, desc=desc, basename=mod.replace(f".{ext}", "") + ) + ) + if num_modules >= min_num_modules_for_topic: + topicsfile.writelines( + [moduletopics_tmpl.substitute(key=key, name=key.replace("_", " "))] + ) + if ext == "html": + keyfile.write("\n") + else: + keyfile.write("\n") + # link to the keywords index + # TODO: the labels in keywords index are with spaces and capitals + # this should be probably changed to lowercase with underscores + if ext == "html": + keyfile.write( + "

    See also the corresponding keyword" + ' {key}' + " for additional references.".format(key=key.replace("_", " ")) + ) + else: + # expecting markdown + keyfile.write( + "*See also the corresponding keyword" + " [{key}](keywords.md#{key})" + " for additional references.*\n".format(key=key.replace("_", " ")) + ) + + write_footer(keyfile, f"index.{ext}", year, template=ext) + if ext == "html": + topicsfile.write("\n") + write_footer(topicsfile, f"index.{ext}", year, template=ext) + topicsfile.close() + + +if __name__ == "__main__": + from build import ( + grass_version, + write_footer, + ) + + build_topics("html") + + build_topics("md") diff --git a/man/mkdocs/grassdocs.css b/man/mkdocs/grassdocs.css new file mode 100644 index 00000000000..e42cc692fd4 --- /dev/null +++ b/man/mkdocs/grassdocs.css @@ -0,0 +1,21 @@ +:root > * { + --md-primary-fg-color: #088B36; + --md-primary-fg-color--light: #088B36; + --md-primary-fg-color--dark: #088B36; + --md-footer-bg-color: #088B36; + --md-footer-bg-color--light: #088B36; + --md-footer-bg-color--dark: #088B36; +} + +.md-header__button.md-logo { + margin-top: 0; + margin-bottom: 0; + padding-top: 0; + padding-bottom: 0; +} + +.md-header__button.md-logo img, +.md-header__button.md-logo svg { + height: 70%; + width: 70%; +} diff --git a/man/mkdocs/mkdocs.yml b/man/mkdocs/mkdocs.yml new file mode 100644 index 00000000000..bd01d13a791 --- /dev/null +++ b/man/mkdocs/mkdocs.yml @@ -0,0 +1,43 @@ +--- +site_name: !ENV SITE_NAME +site_url: https://grass.osgeo.org/grass-stable/manuals/ +docs_dir: source +extra: + homepage: ./index.html +theme: + name: material + custom_dir: overrides + language: en + logo: grass_logo.png + features: + - content.code.copy + - navigation.footer + palette: + primary: custom +copyright: !ENV COPYRIGHT +extra_css: + - grassdocs.css +plugins: + - search + - glightbox +use_directory_urls: false +nav: + - Startup: grass.md + - Databases: database.md + - Display: display.md + - General: general.md + - Imagery: imagery.md + - Keywords: keywords.md + - Misc: miscellaneous.md + - Raster: raster.md + - Raster 3D: raster3d.md + - SQL: sql.md + - Temporal: temporal.md + - Variables: variables.md + - Vector: vector.md +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences + - attr_list + - md_in_html diff --git a/man/mkdocs/overrides/partials/footer.html b/man/mkdocs/overrides/partials/footer.html new file mode 100644 index 00000000000..a713e2e6db8 --- /dev/null +++ b/man/mkdocs/overrides/partials/footer.html @@ -0,0 +1,100 @@ + + + +

    diff --git a/man/mkdocs/requirements.txt b/man/mkdocs/requirements.txt new file mode 100644 index 00000000000..36f78605155 --- /dev/null +++ b/man/mkdocs/requirements.txt @@ -0,0 +1,5 @@ +mkdocs +mkdocs-glightbox +mkdocs-material +pymdown-extensions +pyyaml-env-tag diff --git a/man/parser_standard_options.py b/man/parser_standard_options.py index ad027a1b6b4..5ff79455eef 100644 --- a/man/parser_standard_options.py +++ b/man/parser_standard_options.py @@ -10,13 +10,6 @@ from urllib.request import urlopen -from build_html import ( - header1_tmpl, - grass_version, - headerpso_tmpl, - write_html_footer, -) - def parse_options(lines, startswith="Opt"): def split_in_groups(lines): @@ -132,6 +125,21 @@ def csv(self, delimiter=";", endline="\n"): ) return endline.join(csv) + def markdown(self, endline="\n"): + """Return a Markdown table with the options""" + # write header + md = ["| " + " | ".join(self.columns) + " |"] + md.append("| " + " | ".join(len(x) * "-" for x in self.columns) + " |") + + # write body + for optname, options in self.options: + row = "| {0} ".format(optname) + for col in self.columns: + row += "| {0} ".format(options.get(col, "")) + md.append(row + "|") + + return endline.join(md) + def html(self, endline="\n", indent=" ", toptions="border=1"): """Return a HTML table with the options""" html = ["".format(" " + toptions if toptions else "")] @@ -161,9 +169,10 @@ def _repr_html_(self): if __name__ == "__main__": URL = ( - "https://trac.osgeo.org/grass/browser/grass/" - "trunk/lib/gis/parser_standard_options.c?format=txt" + "https://raw.githubusercontent.com/OSGeo/grass/main/" + "lib/gis/parser_standard_options.c" ) + parser = argparse.ArgumentParser( description="Extract GRASS default options from link." ) @@ -172,7 +181,7 @@ def _repr_html_(self): "--format", default="html", dest="format", - choices=["html", "csv", "grass"], + choices=["html", "csv", "grass", "markdown"], help="Define the output format", ) parser.add_argument( @@ -220,21 +229,45 @@ def _repr_html_(self): options = OptTable(parse_options(cfile.readlines(), startswith=args.startswith)) outform = args.format - if outform in {"csv", "html"}: + if outform in ("csv", "html", "markdown"): print(getattr(options, outform)(), file=args.output) args.output.close() else: year = os.getenv("VERSION_DATE") name = args.output.name args.output.close() - topicsfile = open(name, "w") - topicsfile.write( - header1_tmpl.substitute( - title="GRASS GIS " - "%s Reference Manual: Parser standard options index" % grass_version - ) + + def write_output(ext): + with open(name, "w") as outfile: + outfile.write( + header1_tmpl.substitute( + title=f"GRASS GIS {grass_version} Reference Manual: " + "Parser standard options index" + ) + ) + outfile.write(headerpso_tmpl) + if ext == "html": + outfile.write(options.html(toptions=args.htmlparmas)) + else: + outfile.write(options.markdown()) + write_footer(outfile, f"index.{ext}", year, template=ext) + + from build import ( + grass_version, + write_footer, ) - topicsfile.write(headerpso_tmpl) - topicsfile.write(options.html(toptions=args.htmlparmas)) - write_html_footer(topicsfile, "index.html", year) - topicsfile.close() + + ext = os.path.splitext(name)[1][1:] + + if ext == "html": + from build_html import ( + header1_tmpl, + headerpso_tmpl, + ) + else: + from build_md import ( + header1_tmpl, + headerpso_tmpl, + ) + + write_output(ext) # html or md diff --git a/raster/Makefile b/raster/Makefile index 91ff54d0863..a8f00ee9c7c 100644 --- a/raster/Makefile +++ b/raster/Makefile @@ -147,6 +147,10 @@ htmldir: parsubdirs $(HTMLDIR)/r.in.png: # no-op - override Html.make rule for .png image files +$(MDDIR)/source/r.in.png: + # no-op - override Html.make rule for .png image files $(HTMLDIR)/r.out.png: # no-op - override Html.make rule for .png image files +$(MDDIR)/source/r.out.png: + # no-op - override Html.make rule for .png image files diff --git a/utils/Makefile b/utils/Makefile index fdff52b8e3d..8137e1b7e63 100644 --- a/utils/Makefile +++ b/utils/Makefile @@ -5,13 +5,19 @@ SUBDIRS = timer g.html2man include $(MODULE_TOPDIR)/include/Make/Dir.make include $(MODULE_TOPDIR)/include/Make/Compile.make -default: parsubdirs $(UTILSDIR)/mkhtml.py \ +default: parsubdirs $(UTILSDIR)/mkdocs.py $(UTILSDIR)/mkhtml.py $(UTILSDIR)/mkmarkdown.py \ $(UTILSDIR)/generate_last_commit_file.py \ $(UTILSDIR)/g.echo$(EXE) +$(UTILSDIR)/mkdocs.py: mkdocs.py + $(INSTALL) $< $@ + $(UTILSDIR)/mkhtml.py: mkhtml.py $(INSTALL) $< $@ +$(UTILSDIR)/mkmarkdown.py: mkmarkdown.py + $(INSTALL) $< $@ + $(UTILSDIR)/generate_last_commit_file.py: generate_last_commit_file.py $(INSTALL) $< $@ diff --git a/utils/mkdocs.py b/utils/mkdocs.py new file mode 100644 index 00000000000..1323fe06229 --- /dev/null +++ b/utils/mkdocs.py @@ -0,0 +1,435 @@ +# common functions used by mkmarkdown.py and mkhtml.py + +import sys +import os +import json +import subprocess +import re +import urllib.parse as urlparse +from http import HTTPStatus +from pathlib import Path +from datetime import datetime +from urllib import request as urlrequest +from urllib.error import HTTPError, URLError + +try: + import grass.script as gs +except ImportError: + # During compilation GRASS GIS + gs = None + +from generate_last_commit_file import COMMIT_DATE_FORMAT + +HEADERS = { + "User-Agent": "Mozilla/5.0", +} +HTTP_STATUS_CODES = list(HTTPStatus) + +top_dir = os.path.abspath(os.getenv("MODULE_TOPDIR")) + + +def read_file(name): + try: + return Path(name).read_text() + except OSError: + return "" + + +def get_version_branch(major_version, addons_git_repo_url): + """Check if version branch for the current GRASS version exists, + if not, take branch for the previous version + For the official repo we assume that at least one version branch is present + + :param major_version int: GRASS GIS major version + :param addons_git_repo_url str: Addons Git ropository URL + + :return version_branch str: version branch + """ + version_branch = f"grass{major_version}" + if gs: + branch = gs.Popen( + [ + "git", + "ls-remote", + "--heads", + addons_git_repo_url, + f"refs/heads/{version_branch}", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + branch, stderr = branch.communicate() + if stderr: + gs.fatal( + _( + "Failed to get branch from the Git repository" + " <{repo_path}>.\n{error}" + ).format( + repo_path=addons_git_repo_url, + error=gs.decode(stderr), + ) + ) + if version_branch not in gs.decode(branch): + version_branch = "grass{}".format(int(major_version) - 1) + return version_branch + + +def has_src_code_git(src_dir): + """Has core module or addon source code Git + + :param str src_dir: core module or addon root directory + + :return subprocess.CompletedProcess or None: subprocess.CompletedProcess + if core module or addon + source code has Git + """ + actual_dir = Path.cwd() + os.chdir(src_dir) + try: + process_result = subprocess.run( + [ + "git", + "log", + "-1", + f"--format=%H,{COMMIT_DATE_FORMAT}", + src_dir, + ], + capture_output=True, + ) # --format=%H,COMMIT_DATE_FORMAT commit hash,author date + os.chdir(actual_dir) + return process_result if process_result.returncode == 0 else None + except FileNotFoundError: + os.chdir(actual_dir) + return None + + +def get_last_git_commit(src_dir, top_dir, pgm, addon_path, major_version): + """Get last module/addon git commit + :param str src_dir: module/addon source dir + :param str top_dir: top source dir + :param str pgm: program name + :param str addon_path: addon path + :param str major_version: major GRASS version + + :return dict git_log: dict with key commit and date, if not + possible download commit from GitHub REST API + server values of keys have "unknown" string + """ + process_result = has_src_code_git(src_dir=src_dir) + if process_result: + return parse_git_commit( + commit=process_result.stdout.decode(), + src_dir=src_dir, + ) + if gs: + # Addons installation + return get_git_commit_from_rest_api_for_addon_repo( + addon_path=addon_path, src_dir=src_dir, pgm=pgm, major_version=major_version + ) + # During GRASS GIS compilation from source code without Git + return get_git_commit_from_file(src_dir=src_dir, pgm=pgm) + + +def parse_git_commit( + commit, + src_dir, + git_log=None, +): + """Parse Git commit + + :param str commit: commit message + :param str src_dir: addon source dir + :param dict git_log: dict which store last commit and commnit + date + + :return dict git_log: dict which store last commit and commnit date + """ + if not git_log: + git_log = get_default_git_log(src_dir=src_dir) + if commit: + git_log["commit"], commit_date = commit.strip().split(",") + git_log["date"] = format_git_commit_date_from_local_git( + commit_datetime=commit_date, + ) + return git_log + + +def get_default_git_log(src_dir, datetime_format="%A %b %d %H:%M:%S %Y"): + """Get default Git commit and commit date, when getting commit from + local Git, local JSON file and remote GitHub REST API server wasn't + successful. + + :param str src_dir: addon source dir + :param str datetime_format: output commit datetime format + e.g. Sunday Jan 16 23:09:35 2022 + + :return dict: dict which store last commit and commnit date + """ + return { + "commit": "unknown", + "date": datetime.fromtimestamp(os.path.getmtime(src_dir)).strftime( + datetime_format + ), + } + + +def format_git_commit_date_from_local_git( + commit_datetime, datetime_format="%A %b %d %H:%M:%S %Y" +): + """Format datetime from local Git or JSON file + + :param str commit_datetime: commit datetime + :param str datetime_format: output commit datetime format + e.g. Sunday Jan 16 23:09:35 2022 + + :return str: output formatted commit datetime + """ + try: + date = datetime.fromisoformat( + commit_datetime, + ) + except ValueError: + if commit_datetime.endswith("Z"): + # Python 3.10 and older does not support Z in time, while recent versions + # of Git (2.45.1) use it. Try to help the parsing if Z is in the string. + date = datetime.fromisoformat(commit_datetime[:-1] + "+00:00") + else: + raise + return date.strftime(datetime_format) + + +def get_git_commit_from_rest_api_for_addon_repo( + addon_path, + src_dir, + pgm, + major_version, + git_log=None, +): + """Get Git commit from remote GitHub REST API for addon repository + + :param str addon_path: addon path + :param str src_dir: addon source dir + :param str pgm: program name + :param major_version int: GRASS GIS major version + :param dict git_log: dict which store last commit and commnit date + + :return dict git_log: dict which store last commit and commnit date + """ + # Accessed date time if getting commit from GitHub REST API wasn't successful + if not git_log: + git_log = get_default_git_log(src_dir=src_dir) + if addon_path is not None: + grass_addons_url = ( + "https://api.github.com/repos/osgeo/grass-addons/commits?" + "path={path}&page=1&per_page=1&sha=grass{major}".format( + path=addon_path, + major=major_version, + ) + ) # sha=git_branch_name + + response = download_git_commit( + url=grass_addons_url, + pgm=pgm, + response_format="application/json", + ) + if response: + commit = json.loads(response.read()) + if commit: + git_log["commit"] = commit[0]["sha"] + git_log["date"] = format_git_commit_date_from_rest_api( + commit_datetime=commit[0]["commit"]["author"]["date"], + ) + return git_log + + +def get_git_commit_from_file( + src_dir, + pgm, + git_log=None, +): + """Get Git commit from JSON file + + :param str src_dir: addon source dir + :param str pgm: program name + :param dict git_log: dict which store last commit and commnit date + + :return dict git_log: dict which store last commit and commnit date + """ + # Accessed date time if getting commit from JSON file wasn't successful + if not git_log: + git_log = get_default_git_log(src_dir=src_dir) + json_file_path = os.path.join( + top_dir, + "core_modules_with_last_commit.json", + ) + if os.path.exists(json_file_path): + with open(json_file_path) as f: + core_modules_with_last_commit = json.load(f) + if pgm in core_modules_with_last_commit: + core_module = core_modules_with_last_commit[pgm] + git_log["commit"] = core_module["commit"] + git_log["date"] = format_git_commit_date_from_local_git( + commit_datetime=core_module["date"], + ) + return git_log + + +def download_git_commit(url, pgm, response_format, *args, **kwargs): + """Download module/addon last commit from GitHub API + + :param str url: url address + :param str pgm: program name + :param str response_format: content type + + :return urllib.request.urlopen or None response: response object or + None + """ + try: + response = urlopen(url, *args, **kwargs) + if response.code != 200: + index = HTTP_STATUS_CODES.index(response.code) + desc = HTTP_STATUS_CODES[index].description + gs.fatal( + _( + "Download commit from <{url}>, return status code {code}, {desc}" + ).format( + url=url, + code=response.code, + desc=desc, + ), + ) + if response_format not in response.getheader("Content-Type"): + gs.fatal( + _( + "Wrong downloaded commit file format. " + "Check url <{url}>. Allowed file format is " + "{response_format}." + ).format( + url=url, + response_format=response_format, + ), + ) + return response + except HTTPError as err: + gs.warning( + _( + "The download of the commit from the GitHub API " + "server wasn't successful, <{}>. Commit and commit " + "date will not be included in the <{}> addon html manual " + "page." + ).format(err.msg, pgm), + ) + except URLError: + gs.warning( + _( + "Download file from <{url}>, failed. Check internet " + "connection. Commit and commit date will not be included " + "in the <{pgm}> addon manual page." + ).format(url=url, pgm=pgm), + ) + + +def format_git_commit_date_from_rest_api( + commit_datetime, datetime_format="%A %b %d %H:%M:%S %Y" +): + """Format datetime from remote GitHub REST API + + :param str commit_datetime: commit datetime + :param str datetime_format: output commit datetime format + e.g. Sunday Jan 16 23:09:35 2022 + + :return str: output formatted commit datetime + """ + return datetime.strptime( + commit_datetime, + "%Y-%m-%dT%H:%M:%SZ", # ISO 8601 YYYY-MM-DDTHH:MM:SSZ + ).strftime(datetime_format) + + +def urlopen(url, *args, **kwargs): + """Wrapper around urlopen. Same function as 'urlopen', but with the + ability to define headers. + """ + request = urlrequest.Request(url, headers=HEADERS) + return urlrequest.urlopen(request, *args, **kwargs) + + +def get_addon_path(base_url, pgm, major_version): + """Check if pgm is in the addons list and get addon path + + Make or update list of the official addons source + code paths g.extension prefix parameter plus /grass-addons directory + using Git repository + + :param str base_url: base URL + :param str pgm: program name + :param str major_version: GRASS major version + + :return str|None: pgm path if pgm is addon else None + """ + addons_base_dir = os.getenv("GRASS_ADDON_BASE") + if addons_base_dir and major_version: + grass_addons_dir = Path(addons_base_dir) / "grass-addons" + if gs: + call = gs.call + popen = gs.Popen + fatal = gs.fatal + else: + call = subprocess.call + popen = subprocess.Popen + fatal = sys.stderr.write + addons_branch = get_version_branch( + major_version=major_version, + addons_git_repo_url=urlparse.urljoin(base_url, "grass-addons/"), + ) + if not Path(addons_base_dir).exists(): + Path(addons_base_dir).mkdir(parents=True, exist_ok=True) + if not grass_addons_dir.exists(): + call( + [ + "git", + "clone", + "-q", + "--no-checkout", + f"--branch={addons_branch}", + "--filter=blob:none", + urlparse.urljoin(base_url, "grass-addons/"), + ], + cwd=addons_base_dir, + ) + addons_file_list = popen( + ["git", "ls-tree", "--name-only", "-r", addons_branch], + cwd=grass_addons_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + addons_file_list, stderr = addons_file_list.communicate() + if stderr: + message = ( + "Failed to get addons files list from the" + " Git repository <{repo_path}>.\n{error}" + ) + if gs: + fatal( + _( + message, + ).format( + repo_path=grass_addons_dir, + error=gs.decode(stderr), + ) + ) + else: + message += "\n" + fatal( + message.format( + repo_path=grass_addons_dir, + error=stderr.decode(), + ) + ) + addon_paths = re.findall( + rf".*{pgm}*.", + gs.decode(addons_file_list) if gs else addons_file_list.decode(), + ) + for addon_path in addon_paths: + if pgm == Path(addon_path).name: + return addon_path diff --git a/utils/mkhtml.py b/utils/mkhtml.py index 39a5e6e26eb..363c513a7ed 100644 --- a/utils/mkhtml.py +++ b/utils/mkhtml.py @@ -16,22 +16,16 @@ # ############################################################################# -import http import sys import os import string import re from datetime import datetime import locale -import json -import pathlib -import subprocess -from pathlib import Path from html.parser import HTMLParser from urllib import request as urlrequest -from urllib.error import HTTPError, URLError import urllib.parse as urlparse try: @@ -40,52 +34,13 @@ # During compilation GRASS GIS gs = None -from generate_last_commit_file import COMMIT_DATE_FORMAT - -HEADERS = { - "User-Agent": "Mozilla/5.0", -} -HTTP_STATUS_CODES = list(http.HTTPStatus) - - -def get_version_branch(major_version, addons_git_repo_url): - """Check if version branch for the current GRASS version exists, - if not, take branch for the previous version - For the official repo we assume that at least one version branch is present - - :param major_version int: GRASS GIS major version - :param addons_git_repo_url str: Addons Git ropository URL - - :return version_branch str: version branch - """ - version_branch = f"grass{major_version}" - if gs: - branch = gs.Popen( - [ - "git", - "ls-remote", - "--heads", - addons_git_repo_url, - f"refs/heads/{version_branch}", - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - branch, stderr = branch.communicate() - if stderr: - gs.fatal( - _( - "Failed to get branch from the Git repository" - " <{repo_path}>.\n{error}" - ).format( - repo_path=addons_git_repo_url, - error=gs.decode(stderr), - ) - ) - if version_branch not in gs.decode(branch): - version_branch = "grass{}".format(int(major_version) - 1) - return version_branch - +from mkdocs import ( + read_file, + get_version_branch, + get_last_git_commit, + top_dir as topdir, + get_addon_path, +) grass_version = os.getenv("VERSION_NUMBER", "unknown") trunk_url = "" @@ -125,14 +80,6 @@ def _get_encoding(): return encoding -def urlopen(url, *args, **kwargs): - """Wrapper around urlopen. Same function as 'urlopen', but with the - ability to define headers. - """ - request = urlrequest.Request(url, headers=HEADERS) - return urlrequest.urlopen(request, *args, **kwargs) - - def set_proxy(): """Set proxy""" proxy = os.getenv("GRASS_PROXY") @@ -148,274 +95,6 @@ def set_proxy(): set_proxy() -def download_git_commit(url, response_format, *args, **kwargs): - """Download module/addon last commit from GitHub API - - :param str url: url address - :param str response_format: content type - - :return urllib.request.urlopen or None response: response object or - None - """ - try: - response = urlopen(url, *args, **kwargs) - if response.code != 200: - index = HTTP_STATUS_CODES.index(response.code) - desc = HTTP_STATUS_CODES[index].description - gs.fatal( - _( - "Download commit from <{url}>, return status code {code}, {desc}" - ).format( - url=url, - code=response.code, - desc=desc, - ), - ) - if response_format not in response.getheader("Content-Type"): - gs.fatal( - _( - "Wrong downloaded commit file format. " - "Check url <{url}>. Allowed file format is " - "{response_format}." - ).format( - url=url, - response_format=response_format, - ), - ) - return response - except HTTPError as err: - gs.warning( - _( - "The download of the commit from the GitHub API " - "server wasn't successful, <{}>. Commit and commit " - "date will not be included in the <{}> addon html manual " - "page." - ).format(err.msg, pgm), - ) - except URLError: - gs.warning( - _( - "Download file from <{url}>, failed. Check internet " - "connection. Commit and commit date will not be included " - "in the <{pgm}> addon manual page." - ).format(url=url, pgm=pgm), - ) - - -def get_default_git_log(src_dir, datetime_format="%A %b %d %H:%M:%S %Y"): - """Get default Git commit and commit date, when getting commit from - local Git, local JSON file and remote GitHub REST API server wasn't - successful. - - :param str src_dir: addon source dir - :param str datetime_format: output commit datetime format - e.g. Sunday Jan 16 23:09:35 2022 - - :return dict: dict which store last commit and commnit date - """ - return { - "commit": "unknown", - "date": datetime.fromtimestamp(os.path.getmtime(src_dir)).strftime( - datetime_format - ), - } - - -def parse_git_commit( - commit, - src_dir, - git_log=None, -): - """Parse Git commit - - :param str commit: commit message - :param str src_dir: addon source dir - :param dict git_log: dict which store last commit and commnit - date - - :return dict git_log: dict which store last commit and commnit date - """ - if not git_log: - git_log = get_default_git_log(src_dir=src_dir) - if commit: - git_log["commit"], commit_date = commit.strip().split(",") - git_log["date"] = format_git_commit_date_from_local_git( - commit_datetime=commit_date, - ) - return git_log - - -def get_git_commit_from_file( - src_dir, - git_log=None, -): - """Get Git commit from JSON file - - :param str src_dir: addon source dir - :param dict git_log: dict which store last commit and commnit date - - :return dict git_log: dict which store last commit and commnit date - """ - # Accessed date time if getting commit from JSON file wasn't successful - if not git_log: - git_log = get_default_git_log(src_dir=src_dir) - json_file_path = os.path.join( - topdir, - "core_modules_with_last_commit.json", - ) - if os.path.exists(json_file_path): - with open(json_file_path) as f: - core_modules_with_last_commit = json.load(f) - if pgm in core_modules_with_last_commit: - core_module = core_modules_with_last_commit[pgm] - git_log["commit"] = core_module["commit"] - git_log["date"] = format_git_commit_date_from_local_git( - commit_datetime=core_module["date"], - ) - return git_log - - -def get_git_commit_from_rest_api_for_addon_repo( - addon_path, - src_dir, - git_log=None, -): - """Get Git commit from remote GitHub REST API for addon repository - - :param str addon_path: addon path - :param str src_dir: addon source dir - :param dict git_log: dict which store last commit and commnit date - - :return dict git_log: dict which store last commit and commnit date - """ - # Accessed date time if getting commit from GitHub REST API wasn't successful - if not git_log: - git_log = get_default_git_log(src_dir=src_dir) - if addon_path is not None: - grass_addons_url = ( - "https://api.github.com/repos/osgeo/grass-addons/commits?" - "path={path}&page=1&per_page=1&sha=grass{major}".format( - path=addon_path, - major=major, - ) - ) # sha=git_branch_name - - response = download_git_commit( - url=grass_addons_url, - response_format="application/json", - ) - if response: - commit = json.loads(response.read()) - if commit: - git_log["commit"] = commit[0]["sha"] - git_log["date"] = format_git_commit_date_from_rest_api( - commit_datetime=commit[0]["commit"]["author"]["date"], - ) - return git_log - - -def format_git_commit_date_from_rest_api( - commit_datetime, datetime_format="%A %b %d %H:%M:%S %Y" -): - """Format datetime from remote GitHub REST API - - :param str commit_datetime: commit datetime - :param str datetime_format: output commit datetime format - e.g. Sunday Jan 16 23:09:35 2022 - - :return str: output formatted commit datetime - """ - return datetime.strptime( - commit_datetime, - "%Y-%m-%dT%H:%M:%SZ", # ISO 8601 YYYY-MM-DDTHH:MM:SSZ - ).strftime(datetime_format) - - -def format_git_commit_date_from_local_git( - commit_datetime, datetime_format="%A %b %d %H:%M:%S %Y" -): - """Format datetime from local Git or JSON file - - :param str commit_datetime: commit datetime - :param str datetime_format: output commit datetime format - e.g. Sunday Jan 16 23:09:35 2022 - - :return str: output formatted commit datetime - """ - try: - date = datetime.fromisoformat( - commit_datetime, - ) - except ValueError: - if commit_datetime.endswith("Z"): - # Python 3.10 and older does not support Z in time, while recent versions - # of Git (2.45.1) use it. Try to help the parsing if Z is in the string. - date = datetime.fromisoformat(commit_datetime[:-1] + "+00:00") - else: - raise - return date.strftime(datetime_format) - - -def has_src_code_git(src_dir, is_addon): - """Has core module or addon source code Git - - :param str src_dir: core module or addon root directory - :param bool is_addon: True if it is addon - - :return subprocess.CompletedProcess or None: subprocess.CompletedProcess - if core module or addon - source code has Git - """ - actual_dir = Path.cwd() - if is_addon: - os.chdir(src_dir) - else: - os.chdir(topdir) - try: - process_result = subprocess.run( - [ - "git", - "log", - "-1", - f"--format=%H,{COMMIT_DATE_FORMAT}", - src_dir, - ], - capture_output=True, - ) # --format=%H,COMMIT_DATE_FORMAT commit hash,author date - os.chdir(actual_dir) - return process_result if process_result.returncode == 0 else None - except FileNotFoundError: - os.chdir(actual_dir) - return None - - -def get_last_git_commit(src_dir, addon_path, is_addon): - """Get last module/addon git commit - - :param str src_dir: module/addon source dir - :param str addon_path: addon path - :param bool is_addon: True if it is addon - - :return dict git_log: dict with key commit and date, if not - possible download commit from GitHub REST API - server values of keys have "unknown" string - """ - process_result = has_src_code_git(src_dir=src_dir, is_addon=is_addon) - if process_result: - return parse_git_commit( - commit=process_result.stdout.decode(), - src_dir=src_dir, - ) - if gs: - # Addons installation - return get_git_commit_from_rest_api_for_addon_repo( - addon_path=addon_path, - src_dir=src_dir, - ) - # During GRASS GIS compilation from source code without Git - return get_git_commit_from_file(src_dir=src_dir) - - html_page_footer_pages_path = os.getenv("HTML_PAGE_FOOTER_PAGES_PATH") or "" pgm = sys.argv[1] @@ -509,13 +188,6 @@ def get_last_git_commit(src_dir, addon_path, is_addon): ) -def read_file(name): - try: - return Path(name).read_text() - except OSError: - return "" - - def create_toc(src_data): class MyHTMLParser(HTMLParser): def __init__(self): @@ -677,83 +349,6 @@ def update_toc(data): return "\n".join(ret_data) -def get_addon_path(): - """Check if pgm is in the addons list and get addon path - - Make or update list of the official addons source - code paths g.extension prefix parameter plus /grass-addons directory - using Git repository - - :return str|None: pgm path if pgm is addon else None - """ - addons_base_dir = os.getenv("GRASS_ADDON_BASE") - if addons_base_dir and major: - grass_addons_dir = pathlib.Path(addons_base_dir) / "grass-addons" - if gs: - call = gs.call - popen = gs.Popen - fatal = gs.fatal - else: - call = subprocess.call - popen = subprocess.Popen - fatal = sys.stderr.write - addons_branch = get_version_branch( - major_version=major, - addons_git_repo_url=urlparse.urljoin(base_url, "grass-addons/"), - ) - if not pathlib.Path(addons_base_dir).exists(): - pathlib.Path(addons_base_dir).mkdir(parents=True, exist_ok=True) - if not grass_addons_dir.exists(): - call( - [ - "git", - "clone", - "-q", - "--no-checkout", - f"--branch={addons_branch}", - "--filter=blob:none", - urlparse.urljoin(base_url, "grass-addons/"), - ], - cwd=addons_base_dir, - ) - addons_file_list = popen( - ["git", "ls-tree", "--name-only", "-r", addons_branch], - cwd=grass_addons_dir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - addons_file_list, stderr = addons_file_list.communicate() - if stderr: - message = ( - "Failed to get addons files list from the" - " Git repository <{repo_path}>.\n{error}" - ) - if gs: - fatal( - _( - message, - ).format( - repo_path=grass_addons_dir, - error=gs.decode(stderr), - ) - ) - else: - message += "\n" - fatal( - message.format( - repo_path=grass_addons_dir, - error=stderr.decode(), - ) - ) - addon_paths = re.findall( - rf".*{pgm}*.", - gs.decode(addons_file_list) if gs else addons_file_list.decode(), - ) - for addon_path in addon_paths: - if pgm == pathlib.Path(addon_path).name: - return addon_path - - # process header src_data = read_file(src_file) name = re.search("()", src_data, re.IGNORECASE) @@ -875,7 +470,6 @@ def to_title(name): year = str(datetime.now().year) # check the names of scripts to assign the right folder -topdir = os.path.abspath(os.getenv("MODULE_TOPDIR")) curdir = os.path.abspath(os.path.curdir) if curdir.startswith(topdir + os.path.sep): source_url = trunk_url @@ -887,7 +481,7 @@ def to_title(name): url_source = "" addon_path = None if os.getenv("SOURCE_URL", ""): - addon_path = get_addon_path() + addon_path = get_addon_path(base_url=base_url, pgm=pgm, major_version=major) if addon_path: # Addon is installed from the local dir if os.path.exists(os.getenv("SOURCE_URL")): @@ -918,8 +512,10 @@ def to_title(name): git_commit = get_last_git_commit( src_dir=curdir, + top_dir=topdir, + pgm=pgm, addon_path=addon_path or None, - is_addon=bool(addon_path), + major_version=major, ) if git_commit["commit"] == "unknown": date_tag = "Accessed: {date}".format(date=git_commit["date"]) diff --git a/utils/mkmarkdown.py b/utils/mkmarkdown.py new file mode 100644 index 00000000000..b7f595fc3c4 --- /dev/null +++ b/utils/mkmarkdown.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 + +############################################################################ +# +# MODULE: Builds manual pages (Markdown) +# AUTHOR(S): Markus Neteler +# Glynn Clements +# Martin Landa +# PURPOSE: Create Markdown manual page snippets +# Inspired by mkhtml.py +# COPYRIGHT: (C) 2024 by the GRASS Development Team +# +# This program is free software under the GNU General +# Public License (>=v2). Read the file COPYING that +# comes with GRASS for details. +# +############################################################################# + +import os +import sys +import string +import urllib.parse as urlparse + +try: + import grass.script as gs +except ImportError: + # During compilation GRASS GIS + gs = None + +from mkdocs import ( + read_file, + get_version_branch, + get_last_git_commit, + top_dir, + get_addon_path, +) + + +def parse_source(pgm): + """Parse source code to get source code and log message URLs, + and date time of the last modification. + + :param str pgm: program name + + :return url_source, url_log, date_time + """ + grass_version = os.getenv("VERSION_NUMBER", "unknown") + main_url = "" + addons_url = "" + grass_git_branch = "main" + major, minor, patch = None, None, None + if grass_version != "unknown": + major, minor, patch = grass_version.split(".") + base_url = "https://github.com/OSGeo/" + main_url = urlparse.urljoin( + base_url, + urlparse.urljoin( + "grass/tree/", + grass_git_branch + "/", + ), + ) + addons_url = urlparse.urljoin( + base_url, + urlparse.urljoin( + "grass-addons/tree/", + get_version_branch( + major, + urlparse.urljoin(base_url, "grass-addons/"), + ), + ), + ) + + cur_dir = os.path.abspath(os.path.curdir) + if cur_dir.startswith(top_dir + os.path.sep): + source_url = main_url + pgmdir = cur_dir.replace(top_dir, "").lstrip(os.path.sep) + else: + # addons + source_url = addons_url + pgmdir = os.path.sep.join(cur_dir.split(os.path.sep)[-3:]) + + url_source = "" + addon_path = None + if os.getenv("SOURCE_URL", ""): + addon_path = get_addon_path(base_url=base_url, pgm=pgm, major_version=major) + if addon_path: + # Addon is installed from the local dir + if os.path.exists(os.getenv("SOURCE_URL")): + url_source = urlparse.urljoin( + addons_url, + addon_path, + ) + else: + url_source = urlparse.urljoin( + os.environ["SOURCE_URL"].split("src")[0], + addon_path, + ) + else: + url_source = urlparse.urljoin(source_url, pgmdir) + if sys.platform == "win32": + url_source = url_source.replace(os.path.sep, "/") + + # Process Source code section + branches = "branches" + tree = "tree" + commits = "commits" + + if branches in url_source: + url_log = url_source.replace(branches, commits) + url_source = url_source.replace(branches, tree) + else: + url_log = url_source.replace(tree, commits) + + git_commit = get_last_git_commit( + src_dir=cur_dir, + top_dir=top_dir, + pgm=pgm, + addon_path=addon_path or None, + major_version=major, + ) + if git_commit["commit"] == "unknown": + date_tag = "Accessed: {date}".format(date=git_commit["date"]) + else: + commit = git_commit["commit"] + date_tag = ( + "Latest change: {date} in commit: " + "[{commit_short}](https://github.com/OSGeo/grass/commit/{commit})".format( + date=git_commit["date"], commit=commit, commit_short=commit[:7] + ) + ) + + return url_source, url_log, date_tag + + +if __name__ == "__main__": + pgm = sys.argv[1] + + src_file = f"{pgm}.md" + tmp_file = f"{pgm}.tmp.md" + + sourcecode = string.Template( + """ +## SOURCE CODE + +Available at: [${PGM} source code](${URL_SOURCE}) +([history](${URL_LOG}))${MD_NEWLINE} +${DATE_TAG} +""" + ) + + # process header/usage generated by --md-description + sys.stdout.write(read_file(tmp_file)) + sys.stdout.write("\n") + # process body + sys.stdout.write(read_file(src_file)) + + # process footer + url_source, url_log, date_tag = parse_source(pgm) + sys.stdout.write( + sourcecode.substitute( + URL_SOURCE=url_source, + PGM=pgm, + URL_LOG=url_log, + DATE_TAG=date_tag, + MD_NEWLINE=" ", + ) + )