From 956736ae029392a53dee2032883aa0f4af728f9d Mon Sep 17 00:00:00 2001 From: sstroemer Date: Sun, 24 Nov 2024 11:33:36 +0100 Subject: [PATCH 1/9] feat: disable automatically creating dynamic docstrings --- src/core.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core.jl b/src/core.jl index 198b9ec1..907576e9 100644 --- a/src/core.jl +++ b/src/core.jl @@ -110,12 +110,12 @@ include("core/unit.jl") include("core/virtual.jl") # Finalize the docstrings of the core components. -_finalize_docstring(Connection) -_finalize_docstring(Decision) -_finalize_docstring(Node) -_finalize_docstring(Profile) -_finalize_docstring(Unit) -_finalize_docstring(Virtual) +# _finalize_docstring(Connection) +# _finalize_docstring(Decision) +# _finalize_docstring(Node) +# _finalize_docstring(Profile) +# _finalize_docstring(Unit) +# _finalize_docstring(Virtual) @recompile_invalidations begin function Base.show(io::IO, @nospecialize(cc::_CoreComponent)) From b6b82d8534d17ef743159b737e11cf161a75fc45 Mon Sep 17 00:00:00 2001 From: sstroemer Date: Sun, 24 Nov 2024 11:33:53 +0100 Subject: [PATCH 2/9] feat: implement `_get_dynamic_documentation` to access docs from outside --- src/utils/docs.jl | 253 ++++++++++++++++++++++++++++------------------ 1 file changed, 155 insertions(+), 98 deletions(-) diff --git a/src/utils/docs.jl b/src/utils/docs.jl index 43df2c01..df3b794e 100644 --- a/src/utils/docs.jl +++ b/src/utils/docs.jl @@ -10,128 +10,185 @@ function _parse_field_docstring(docstring::String) descr = replace(strip(str_descr), "\n" => " ") # Return specs (in correct order) and description. - return ([specs[s] for s in ["mandatory", "values", "unit", "default"]]..., descr) + return Dict( + [s => String(specs[s]) for s in ("mandatory", "values", "unit", "default")]..., + "description" => String(descr), + ) end -function _docs_struct_to_table(datatype::Type) - # Start table with proper header. - table_rows = [ - "| Name | Mandatory | Values | Unit | Default | Description |", - "|:-----|:----------|:-------|:-----|:--------|:------------|", - ] +# function _docs_struct_to_table(datatype::Type) +# # Start table with proper header. +# table_rows = [ +# "| Name | Mandatory | Values | Unit | Default | Description |", +# "|:-----|:----------|:-------|:-----|:--------|:------------|", +# ] + +# # Get proper binding from module, error if structure is unexpected. +# binding = Base.Docs.aliasof(datatype, typeof(datatype)) +# dict = Base.Docs.meta(binding.mod; autoinit=false) +# isnothing(dict) && @critical "Doc error occurred" datatype +# haskey(dict, binding) || @critical "Doc error occurred" datatype dict binding +# multidoc = dict[binding] +# haskey(multidoc.docs, Union{}) || @critical "Doc error occurred" datatype multidoc.docs + +# # Get all fields that have a docstring. +# all_doc_fields = multidoc.docs[Union{}].data[:fields] + +# # Get all fields in the order they are defined (`all_doc_fields` is an unordered dictionary). +# all_fields = fieldnames(datatype) + +# # Create a row for each field, properly splitting the docstring. +# for field in all_fields +# haskey(all_doc_fields, field) || continue +# field_attrs = join(_parse_field_docstring(all_doc_fields[field]), " | ") +# push!(table_rows, "| `$(field)` | $(field_attrs) |") +# end + +# # Join all rows to the string representation of the table and parse it to Markdown. +# return Markdown.parse(join(table_rows, "\n")) +# end + +function _docs_docstr_to_admonition(f_name::String) + obj_cc, obj_type, obj_name = String.(match(r"_([^_]+)_([^_]+)_(.*)!", f_name).captures) - # Get proper binding from module, error if structure is unexpected. - binding = Base.Docs.aliasof(datatype, typeof(datatype)) - dict = Base.Docs.meta(binding.mod; autoinit=false) - isnothing(dict) && @critical "Doc error occurred" datatype - haskey(dict, binding) || @critical "Doc error occurred" datatype dict binding - multidoc = dict[binding] - haskey(multidoc.docs, Union{}) || @critical "Doc error occurred" datatype multidoc.docs - - # Get all fields that have a docstring. - all_doc_fields = multidoc.docs[Union{}].data[:fields] - - # Get all fields in the order they are defined (`all_doc_fields` is an unordered dictionary). - all_fields = fieldnames(datatype) + obj_longtype = + Dict("var" => "variable", "exp" => "expression", "con" => "constraint", "obj" => "objective")[obj_type]::String - # Create a row for each field, properly splitting the docstring. - for field in all_fields - haskey(all_doc_fields, field) || continue - field_attrs = join(_parse_field_docstring(all_doc_fields[field]), " | ") - push!(table_rows, "| `$(field)` | $(field_attrs) |") - end + f_path = "ait-energy/IESopt.jl/tree/main/src/core/$(obj_cc)/$(obj_type)_$(obj_name).jl" - # Join all rows to the string representation of the table and parse it to Markdown. - return Markdown.parse(join(table_rows, "\n")) + # header = """ + # !!! tip "How to?" + # Access this $(obj_longtype) by using: + + # ```julia + # # Julia + # get_component(model, "your_$(obj_cc)").$(obj_type).$(obj_name) + # ``` + + # ```python + # # Python + # model.get_component("your_$(obj_cc)").$(obj_type).$(obj_name) + # ``` + + # You can find the full implementation and all details here: [`$(obj_cc)/$(obj_type)_$(obj_name) @ IESopt.jl`](https://github.com/$(f_path)). + # """ + + # Delete the method signature and restore `math` code block tags from `$$` (from Markdown.parse). + f = getfield(IESopt, Symbol(f_name)) + docstr = string(@eval @doc($(Symbol(f_name)))) + docstr = replace(docstr, r"```\n(?s).*```\n\n" => "") + docstr = replace(docstr, r"\$\$(.*?)\$\$"s => c -> """\n```math\n$(strip(c[3:(end-2)]))\n```\n""") + docstr = String(strip(docstr)) + + return Dict( + "type_long" => obj_longtype, + "type_short" => obj_type, + "name" => obj_name, + "component" => obj_cc, + "docstr" => docstr, + "code_path" => f_path, + ) + # return """ + # !!! details "$obj_name" + # $(docstr) + # """ end -function _docs_docstr_to_admonition(f::Function) - f_name = string(f) - obj_cc, obj_type, obj_name = string.(match(r"_([^_]+)_([^_]+)_(.*)!", f_name).captures) - obj_longtype = - Dict("var" => "variable", "exp" => "expression", "con" => "constraint", "obj" => "objective")[obj_type] - f_path = "ait-energy/IESopt.jl/tree/main/src/core/$(obj_cc)/$(obj_type)_$(obj_name).jl" +# function _docs_make_parameters(datatype::Type) +# return """ +# # Parameters - header = """ - !!! tip "How to?" - Access this $(obj_longtype) by using: - - ```julia - # Julia - get_component(model, "your_$(obj_cc)").$(obj_type).$(obj_name) - ``` - - ```python - # Python - model.get_component("your_$(obj_cc)").$(obj_type).$(obj_name) - ``` - - You can find the full implementation and all details here: [`$(obj_cc)/$(obj_type)_$(obj_name) @ IESopt.jl`](https://github.com/$(f_path)). - """ - - docstr = string(@doc f) - docstr = replace(docstr, r"```(?s).*```" => header) - docstr = replace(docstr, "\n" => "\n ", "\$\$" => "```math") # TODO - - return """ - !!! details "$obj_name" - $(docstr) - """ -end +# $(_docs_struct_to_table(datatype)) +# """ +# end -function _docs_make_parameters(datatype::Type) - return """ - # Parameters +# function _docs_make_model_reference(datatype::Type) +# lc_type = lowercase(string(nameof(datatype))) +# registered_names = string.(names(@__MODULE__; all=true, imported=false)) +# valid_names = filter(n -> startswith(n, "_$(lc_type)_"), registered_names) - $(_docs_struct_to_table(datatype)) - """ -end +# var_names = filter(n -> startswith(n, "_$(lc_type)_var_"), valid_names) +# exp_names = filter(n -> startswith(n, "_$(lc_type)_exp_"), valid_names) +# con_names = filter(n -> startswith(n, "_$(lc_type)_con_"), valid_names) +# obj_names = filter(n -> startswith(n, "_$(lc_type)_obj_"), valid_names) -function _docs_make_model_reference(datatype::Type) - lc_type = lowercase(string(nameof(datatype))) - registered_names = string.(names(@__MODULE__; all=true, imported=false)) - valid_names = filter(n -> startswith(n, "_$(lc_type)_"), registered_names) +# return """ +# # Detailed Model Reference - var_names = filter(n -> startswith(n, "_$(lc_type)_var_"), valid_names) - exp_names = filter(n -> startswith(n, "_$(lc_type)_exp_"), valid_names) - con_names = filter(n -> startswith(n, "_$(lc_type)_con_"), valid_names) - obj_names = filter(n -> startswith(n, "_$(lc_type)_obj_"), valid_names) +# ## Variables - return """ - # Detailed Model Reference +# $(join([_docs_docstr_to_admonition(getfield(@__MODULE__, Symbol(n))) for n in var_names], "\n\n")) - ## Variables +# ## Expressions - $(join([_docs_docstr_to_admonition(getfield(@__MODULE__, Symbol(n))) for n in var_names], "\n\n")) +# $(join([_docs_docstr_to_admonition(getfield(@__MODULE__, Symbol(n))) for n in exp_names], "\n\n")) - ## Expressions +# ## Constraints - $(join([_docs_docstr_to_admonition(getfield(@__MODULE__, Symbol(n))) for n in exp_names], "\n\n")) +# $(join([_docs_docstr_to_admonition(getfield(@__MODULE__, Symbol(n))) for n in con_names], "\n\n")) - ## Constraints +# ## Objectives - $(join([_docs_docstr_to_admonition(getfield(@__MODULE__, Symbol(n))) for n in con_names], "\n\n")) +# $(join([_docs_docstr_to_admonition(getfield(@__MODULE__, Symbol(n))) for n in obj_names], "\n\n")) +# """ +# end - ## Objectives +# function _finalize_docstring(datatype::Type) +# binding = Base.Docs.aliasof(datatype, typeof(datatype)) +# multidoc = Base.Docs.meta(@__MODULE__)[binding] +# old_data = multidoc.docs[Union{}].data - $(join([_docs_docstr_to_admonition(getfield(@__MODULE__, Symbol(n))) for n in obj_names], "\n\n")) - """ -end +# original_docstr = (@doc datatype) +# multidoc.docs[Union{}] = Base.Docs.docstr(""" +# $(original_docstr) -function _finalize_docstring(datatype::Type) - binding = Base.Docs.aliasof(datatype, typeof(datatype)) - multidoc = Base.Docs.meta(@__MODULE__)[binding] - old_data = multidoc.docs[Union{}].data +# $(_docs_make_parameters(datatype)) - original_docstr = (@doc datatype) - multidoc.docs[Union{}] = Base.Docs.docstr(""" - $(original_docstr) +# $(_docs_make_model_reference(datatype)) +# """) +# multidoc.docs[Union{}].data = old_data - $(_docs_make_parameters(datatype)) +# return nothing +# end - $(_docs_make_model_reference(datatype)) - """) - multidoc.docs[Union{}].data = old_data +function _get_dynamic_documentation(datatype::Type) + # Get documentation of the datatype (struct). + binding = Base.Docs.aliasof(datatype, typeof(datatype)) + dict = Base.Docs.meta(binding.mod; autoinit=false) + isnothing(dict) && @critical "Could not create dynamic documentation (code 1)" datatype + haskey(dict, binding) || @critical "Could not create dynamic documentation (code 2)" datatype dict binding + multidoc = dict[binding] + haskey(multidoc.docs, Union{}) || @critical "Could not create dynamic documentation (code 3)" datatype multidoc.docs - return nothing + # Get all fields that have a docstring, and all in order (`all_doc_fields` is an unordered dictionary). + all_doc_fields = multidoc.docs[Union{}].data[:fields] + all_fields = collect(string(fn) for fn in fieldnames(datatype)) + + # Prepare documentation of all functions associated with creating JuMP objects. + lc_type = lowercase(string(nameof(datatype))) + registered_names = string.(names(IESopt; all=true, imported=false)) + valid_names = filter(n -> startswith(n, "_$(lc_type)_"), registered_names) + var_names = filter(n -> startswith(n, "_$(lc_type)_var_"), valid_names) + exp_names = filter(n -> startswith(n, "_$(lc_type)_exp_"), valid_names) + con_names = filter(n -> startswith(n, "_$(lc_type)_con_"), valid_names) + obj_names = filter(n -> startswith(n, "_$(lc_type)_obj_"), valid_names) + all_functions = vcat(var_names, exp_names, con_names, obj_names) + + # Return all information in a dictionary. + return Dict{String, Union{String, Dict, Vector}}( + "docstr_main" => string(@doc(datatype)), + "fields_all" => string.(all_fields), + "fields_documented" => string.(keys(all_doc_fields)), + "docstr_fields" => Dict{String, Dict}( + string(field) => _parse_field_docstring(all_doc_fields[field]) for + field in fieldnames(datatype) if haskey(all_doc_fields, field) + ), + "functions" => Dict{String, Vector{String}}( + "var" => var_names, + "exp" => exp_names, + "con" => con_names, + "obj" => obj_names, + ), + "docstr_functions" => Dict{String, Dict}(f => _docs_docstr_to_admonition(f) for f in all_functions), + ) end From 2dc9cec6f478d969d0efd0a7b9325d1f0a126062 Mon Sep 17 00:00:00 2001 From: sstroemer Date: Sun, 24 Nov 2024 12:58:04 +0100 Subject: [PATCH 3/9] fix: proper runtime access from Python and convert all docstrings to String --- src/utils/docs.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/utils/docs.jl b/src/utils/docs.jl index df3b794e..d1c22e68 100644 --- a/src/utils/docs.jl +++ b/src/utils/docs.jl @@ -74,11 +74,10 @@ function _docs_docstr_to_admonition(f_name::String) # """ # Delete the method signature and restore `math` code block tags from `$$` (from Markdown.parse). - f = getfield(IESopt, Symbol(f_name)) docstr = string(@eval @doc($(Symbol(f_name)))) docstr = replace(docstr, r"```\n(?s).*```\n\n" => "") docstr = replace(docstr, r"\$\$(.*?)\$\$"s => c -> """\n```math\n$(strip(c[3:(end-2)]))\n```\n""") - docstr = String(strip(docstr)) + docstr = string(strip(docstr)) return Dict( "type_long" => obj_longtype, @@ -176,7 +175,7 @@ function _get_dynamic_documentation(datatype::Type) # Return all information in a dictionary. return Dict{String, Union{String, Dict, Vector}}( - "docstr_main" => string(@doc(datatype)), + "docstr_main" => string(@eval @doc($(Symbol(nameof(datatype))))), "fields_all" => string.(all_fields), "fields_documented" => string.(keys(all_doc_fields)), "docstr_fields" => Dict{String, Dict}( From 67a9027c293b14875e66ee5a026089be295b3c6e Mon Sep 17 00:00:00 2001 From: sstroemer Date: Sun, 24 Nov 2024 13:04:40 +0100 Subject: [PATCH 4/9] fix: ensure "String" to prevent returning "DocStr" instead --- src/utils/docs.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/docs.jl b/src/utils/docs.jl index d1c22e68..3ce13e9d 100644 --- a/src/utils/docs.jl +++ b/src/utils/docs.jl @@ -74,7 +74,7 @@ function _docs_docstr_to_admonition(f_name::String) # """ # Delete the method signature and restore `math` code block tags from `$$` (from Markdown.parse). - docstr = string(@eval @doc($(Symbol(f_name)))) + docstr = String(string(@eval @doc($(Symbol(f_name))))) docstr = replace(docstr, r"```\n(?s).*```\n\n" => "") docstr = replace(docstr, r"\$\$(.*?)\$\$"s => c -> """\n```math\n$(strip(c[3:(end-2)]))\n```\n""") docstr = string(strip(docstr)) @@ -175,7 +175,7 @@ function _get_dynamic_documentation(datatype::Type) # Return all information in a dictionary. return Dict{String, Union{String, Dict, Vector}}( - "docstr_main" => string(@eval @doc($(Symbol(nameof(datatype))))), + "docstr_main" => String(string(@eval @doc($(Symbol(nameof(datatype)))))), "fields_all" => string.(all_fields), "fields_documented" => string.(keys(all_doc_fields)), "docstr_fields" => Dict{String, Dict}( From 6984e67508d7bd52496773f7d49d23df7ae8317c Mon Sep 17 00:00:00 2001 From: sstroemer Date: Sun, 24 Nov 2024 13:26:43 +0100 Subject: [PATCH 5/9] fix: two different paths of conversion are necessary (for calling from Julia and Python) --- src/utils/docs.jl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/utils/docs.jl b/src/utils/docs.jl index 3ce13e9d..99a4398e 100644 --- a/src/utils/docs.jl +++ b/src/utils/docs.jl @@ -1,3 +1,11 @@ +function py_and_jl_convert(docstr::Docs.DocStr) + return string(join(docstr.text), '\n')::String +end + +function py_and_jl_convert(docstr::Markdown.MD) + return string(docstr)::String +end + function _parse_field_docstring(docstring::String) # Split docstring into "spec" fields and "description". rm = match(r"```(\{.*?\})```((?s).*)", docstring) @@ -74,7 +82,7 @@ function _docs_docstr_to_admonition(f_name::String) # """ # Delete the method signature and restore `math` code block tags from `$$` (from Markdown.parse). - docstr = String(string(@eval @doc($(Symbol(f_name))))) + docstr = py_and_jl_convert(@eval @doc($(Symbol(f_name)))) docstr = replace(docstr, r"```\n(?s).*```\n\n" => "") docstr = replace(docstr, r"\$\$(.*?)\$\$"s => c -> """\n```math\n$(strip(c[3:(end-2)]))\n```\n""") docstr = string(strip(docstr)) @@ -175,7 +183,7 @@ function _get_dynamic_documentation(datatype::Type) # Return all information in a dictionary. return Dict{String, Union{String, Dict, Vector}}( - "docstr_main" => String(string(@eval @doc($(Symbol(nameof(datatype)))))), + "docstr_main" => py_and_jl_convert(@eval @doc($(Symbol(nameof(datatype))))), "fields_all" => string.(all_fields), "fields_documented" => string.(keys(all_doc_fields)), "docstr_fields" => Dict{String, Dict}( From eea411c613090420f90113dc70ea1da766eea96f Mon Sep 17 00:00:00 2001 From: sstroemer Date: Sun, 24 Nov 2024 13:38:55 +0100 Subject: [PATCH 6/9] fix: prevent re-integration of ``` math blocks, since sphinx cannot handle them --- src/utils/docs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/docs.jl b/src/utils/docs.jl index 99a4398e..9ec8b004 100644 --- a/src/utils/docs.jl +++ b/src/utils/docs.jl @@ -84,7 +84,7 @@ function _docs_docstr_to_admonition(f_name::String) # Delete the method signature and restore `math` code block tags from `$$` (from Markdown.parse). docstr = py_and_jl_convert(@eval @doc($(Symbol(f_name)))) docstr = replace(docstr, r"```\n(?s).*```\n\n" => "") - docstr = replace(docstr, r"\$\$(.*?)\$\$"s => c -> """\n```math\n$(strip(c[3:(end-2)]))\n```\n""") + # docstr = replace(docstr, r"\$\$(.*?)\$\$"s => c -> """\n```math\n$(strip(c[3:(end-2)]))\n```\n""") docstr = string(strip(docstr)) return Dict( From d7a3a0ca131f0e190799f41ebf23879285be8433 Mon Sep 17 00:00:00 2001 From: sstroemer Date: Sun, 24 Nov 2024 13:44:03 +0100 Subject: [PATCH 7/9] fix: use Markdown.parse to ensure proper rendering of DocStr when calling from Python --- src/utils/docs.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/docs.jl b/src/utils/docs.jl index 9ec8b004..cb816b07 100644 --- a/src/utils/docs.jl +++ b/src/utils/docs.jl @@ -1,5 +1,5 @@ function py_and_jl_convert(docstr::Docs.DocStr) - return string(join(docstr.text), '\n')::String + return string(Markdown.parse(join(docstr.text), '\n'))::String end function py_and_jl_convert(docstr::Markdown.MD) @@ -84,7 +84,6 @@ function _docs_docstr_to_admonition(f_name::String) # Delete the method signature and restore `math` code block tags from `$$` (from Markdown.parse). docstr = py_and_jl_convert(@eval @doc($(Symbol(f_name)))) docstr = replace(docstr, r"```\n(?s).*```\n\n" => "") - # docstr = replace(docstr, r"\$\$(.*?)\$\$"s => c -> """\n```math\n$(strip(c[3:(end-2)]))\n```\n""") docstr = string(strip(docstr)) return Dict( From d0a37e6bc3ecff203c2517d8f4f7986cfcbf8238 Mon Sep 17 00:00:00 2001 From: sstroemer Date: Sun, 24 Nov 2024 13:47:12 +0100 Subject: [PATCH 8/9] fix: misplaced bracket --- src/utils/docs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/docs.jl b/src/utils/docs.jl index cb816b07..06b1ef09 100644 --- a/src/utils/docs.jl +++ b/src/utils/docs.jl @@ -1,5 +1,5 @@ function py_and_jl_convert(docstr::Docs.DocStr) - return string(Markdown.parse(join(docstr.text), '\n'))::String + return string(Markdown.parse(join(docstr.text, '\n')))::String end function py_and_jl_convert(docstr::Markdown.MD) From 1d003c92387b458e9b7245d71485133cdfbac36d Mon Sep 17 00:00:00 2001 From: sstroemer Date: Sun, 24 Nov 2024 15:04:44 +0100 Subject: [PATCH 9/9] fix: py_and_jl_convert was not tagged private --- src/utils/docs.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/utils/docs.jl b/src/utils/docs.jl index 06b1ef09..e49a52a9 100644 --- a/src/utils/docs.jl +++ b/src/utils/docs.jl @@ -1,8 +1,8 @@ -function py_and_jl_convert(docstr::Docs.DocStr) +function _py_and_jl_convert(docstr::Docs.DocStr) return string(Markdown.parse(join(docstr.text, '\n')))::String end -function py_and_jl_convert(docstr::Markdown.MD) +function _py_and_jl_convert(docstr::Markdown.MD) return string(docstr)::String end @@ -82,7 +82,7 @@ function _docs_docstr_to_admonition(f_name::String) # """ # Delete the method signature and restore `math` code block tags from `$$` (from Markdown.parse). - docstr = py_and_jl_convert(@eval @doc($(Symbol(f_name)))) + docstr = _py_and_jl_convert(@eval @doc($(Symbol(f_name)))) docstr = replace(docstr, r"```\n(?s).*```\n\n" => "") docstr = string(strip(docstr)) @@ -182,7 +182,7 @@ function _get_dynamic_documentation(datatype::Type) # Return all information in a dictionary. return Dict{String, Union{String, Dict, Vector}}( - "docstr_main" => py_and_jl_convert(@eval @doc($(Symbol(nameof(datatype))))), + "docstr_main" => _py_and_jl_convert(@eval @doc($(Symbol(nameof(datatype))))), "fields_all" => string.(all_fields), "fields_documented" => string.(keys(all_doc_fields)), "docstr_fields" => Dict{String, Dict}(