Footnote (#230)

* initial footnote capacity

* footnote and fixes
- added footnote capacity
- more tests resulting from judoctemplates updating
- more fixes as a result too

* version bump (patch)

Project.toml

@@ -1,7 +1,7 @@
 name = "JuDoc"
 uuid = "4ca9428c-4c75-11e9-2efb-bf5cb6c1e8f8"
 authors = ["Thibaut Lienart <[email protected]>"]
-version = "0.3.1"
+version = "0.3.2"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -10,6 +10,7 @@ DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 JuDocTemplates = "6793090a-55ae-11e9-0511-73b91164f4ea"
 LiveServer = "16fef848-5104-11e9-1b77-fb7a48bbb589"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [compat]

src/JuDoc.jl

@@ -6,6 +6,7 @@ using Markdown
 using Markdown: htmlesc
 using Dates # see jd_vars
 using DelimitedFiles: readdlm
+using OrderedCollections
 
 import LiveServer
 
@@ -74,7 +75,7 @@ include("parser/lx_tokens.jl")
 include("parser/lx_blocks.jl")
 # > markdown
 include("parser/md_tokens.jl")
-include("parser/md_chars.jl")
+include("parser/md_validate.jl")
 # > html
 include("parser/html_tokens.jl")
 include("parser/html_blocks.jl")

src/converter/fixer.jl

@@ -3,9 +3,9 @@ $(SIGNATURES)
 
 Direct inline-style links are properly processed by Julia's Markdown processor but not:
 
-* `[link title](https://www.google.com "Google's Homepage")`
 * `[link title][some reference]` and later `[some reference]: http://www.reddit.com`
 * `[link title]` and later `[link title]: https://www.mozilla.org`
+* (we don't either) `[link title](https://www.google.com "Google's Homepage")`
 """
 function find_and_fix_md_links(hs::String)::String
     # 1. find all occurences of -- [...]: link
@@ -17,7 +17,10 @@ function find_and_fix_md_links(hs::String)::String
     m_link_defs = collect(eachmatch(r"&#91;((?:(?!&#93;).)*?)&#93;:\s+((?:(?!\<\/p\>)\S)+)", hs))
 
     def_names = [def.captures[1] for def in m_link_defs]
-    def_links = [def.captures[2] for def in m_link_defs]
+
+    # here's a trick, we do NOT use the link caught here; rather we check the dictionary
+    # PAGE_LINK_DEFS as otherwise the link may have been altered by JuDoc (e.g. if has underscores)
+    def_links = [PAGE_LINK_DEFS[def.captures[1]] for def in m_link_defs]
 
     # here we're looking for [id] or [stuff][id] or ![stuff][id] but not [id]:
     m_link_refs = collect(eachmatch(r"(&#33;)?&#91;(.*?)&#93;(?!:)(?:&#91;(.*?)&#93;)?", hs))

src/converter/js_prerender.jl

@@ -98,7 +98,12 @@ function js2html(hs::String, jsbuffer::IOBuffer, matches::Vector{RegexMatch},
     for i ∈ 1:2:length(matches)-1
         mo, mc = matches[i:i+1]
         write(htmls, subs(hs, head, mo.offset - 1))
-        write(htmls, parts[c])
+        pp = strip(parts[c])
+        if startswith(pp, "<pre><code class=\"julia-repl")
+            pp = replace(pp, r"shell&gt;"=>"<span class=hljs-metas>shell&gt;</span>")
+            pp = replace(pp, r"(\(.*?\)) pkg&gt;"=>s"<span class=hljs-metap>\1 pkg&gt;</span>")
+        end
+        write(htmls, pp)
         head = mc.offset + length(mc.match)
         c += 1
     end

src/converter/md.jl

@@ -23,6 +23,8 @@ function convert_md(mds::String, pre_lxdefs::Vector{LxDef}=Vector{LxDef}();
         def_PAGE_HEADERS!()     # all the headers
         def_PAGE_EQREFS!()      # page-specific equation dict (hrefs)
         def_PAGE_BIBREFS!()     # page-specific reference dict (hrefs)
+        def_PAGE_FNREFS!()      # page-specific footnote dict
+        def_PAGE_LINK_DEFS!()   # page-specific link definition candidates [..]: (...)
     end
 
     #
@@ -31,9 +33,10 @@ function convert_md(mds::String, pre_lxdefs::Vector{LxDef}=Vector{LxDef}();
     #
 
     #> 1. Tokenize
-    tokens = find_tokens(mds, MD_TOKENS, MD_1C_TOKENS)
+    tokens  = find_tokens(mds, MD_TOKENS, MD_1C_TOKENS)
+    fn_refs = validate_footnotes!(tokens)
 
-    #> 1'. Find indented blocks
+    #> 1b. Find indented blocks
     tokens = find_indented_blocks(tokens, mds)
 
     #> 2. Open-Close blocks (OCBlocks)
@@ -45,6 +48,8 @@ function convert_md(mds::String, pre_lxdefs::Vector{LxDef}=Vector{LxDef}();
     filter!(τ -> τ.name ∉ L_RETURNS, tokens)
     #>> d. filter out "fake headers" (opening ### that are not at the start of a line)
     filter!(β -> validate_header_block(β), blocks)
+    #>> e. keep track of literal content of possible link definitions to use
+    validate_and_store_link_defs!(blocks)
 
     #> 3. LaTeX commands
     #>> a. find "newcommands", update active blocks/braces
@@ -70,7 +75,7 @@ function convert_md(mds::String, pre_lxdefs::Vector{LxDef}=Vector{LxDef}();
     #
 
     #> 1. Merge all the blocks that will need further processing before insertion
-    blocks2insert = merge_blocks(lxcoms, deactivate_divs(blocks), sp_chars)
+    blocks2insert = merge_blocks(lxcoms, deactivate_divs(blocks), fn_refs, sp_chars)
 
     #> 2. Form intermediate markdown + html
     inter_md, mblocks = form_inter_md(mds, blocks2insert, lxdefs)

src/converter/md_blocks.jl

@@ -7,15 +7,16 @@ Helper function for `convert_inter_html` that processes an extracted block given
 function convert_block(β::AbstractBlock, lxcontext::LxContext)::AS
     # case for special characters / html entities
     β isa HTML_SPCH     && return ifelse(isempty(β.r), β.ss, β.r)
-
     # Return relevant interpolated string based on case
     βn = β.name
     βn ∈  MD_HEADER        && return convert_header(β)
     βn == :CODE_INLINE     && return html_code_inline(content(β) |> htmlesc)
     βn == :CODE_BLOCK_LANG && return convert_code_block(β.ss)
     βn == :CODE_BLOCK_IND  && return convert_indented_code_block(β.ss)
-    βn == :CODE_BLOCK      && return html_code(strip(content(β) |> htmlesc), "{{fill lang}}")
+    βn == :CODE_BLOCK      && return html_code(strip(content(β)), "{{fill lang}}")
     βn == :ESCAPE          && return chop(β.ss, head=3, tail=3)
+    βn == :FOOTNOTE_REF    && return convert_footnote_ref(β)
+    βn == :FOOTNOTE_DEF    && return convert_footnote_def(β, lxcontext)
 
     # Math block --> needs to call further processing to resolve possible latex
     βn ∈ MATH_BLOCKS_NAMES && return convert_math_block(β, lxcontext.lxdefs)
@@ -185,5 +186,60 @@ function convert_indented_code_block(ss::SubString)::String
     # 1. decrease indentation of all lines (either frontal \n\t or \n⎵⎵⎵⎵)
     code = replace(ss, r"\n(?:\t| {4})" => "\n")
     # 2. return; lang is a LOCAL_PAGE_VARS that is julia by default and can be set
-    return html_code(strip(code) |> htmlesc, "{{fill lang}}")
+    return html_code(strip(code), "{{fill lang}}")
+end
+
+"""
+$(SIGNATURES)
+
+Helper function to convert a `[^1]` into a html sup object with appropriate ref and backref.
+"""
+function convert_footnote_ref(β::Token)::String
+    # β.ss is [^id]; extract id
+    id = string(match(r"\[\^(.*?)\]", β.ss).captures[1])
+    # add it to the list of refs unless it's been seen before
+    pos = 0
+    for (i, pri) in enumerate(PAGE_FNREFS)
+        if pri == id
+            pos = i
+            break
+        end
+    end
+    if pos == 0
+        push!(PAGE_FNREFS, id)
+        pos = length(PAGE_FNREFS)
+    end
+    return html_sup("fnref:$id", html_ahref("#fndef:$id", "[$pos]"; class="fnref"))
+end
+
+"""
+$(SIGNATURES)
+
+Helper function to convert a `[^1]: ...` into a html table for the def.
+"""
+function convert_footnote_def(β::OCBlock, lxcontext::LxContext)::String
+    # otok(β) is [^id]:
+    id = match(r"\[\^(.*?)\]:", otok(β).ss).captures[1]
+    pos = 0
+    for (i, pri) in enumerate(PAGE_FNREFS)
+        if pri == id
+            pos = i
+            break
+        end
+    end
+    if pos == 0
+        # this was never referenced before, so probably best not to show it
+        return ""
+    end
+    # need to process the content which could contain stuff
+    ct, _ = convert_md(content(β) * EOS, lxcontext.lxdefs;
+                       isrecursive=true, has_mddefs=false)
+    """
+    <table class="fndef" id="fndef:$id">
+        <tr>
+            <td class=\"fndef-backref\">$(html_ahref("#fnref:$id", "[$pos]"))</td>
+            <td class=\"fndef-content\">$(ct)</td>
+        </tr>
+    </table>
+    """
 end

src/converter/md_utils.jl

@@ -48,27 +48,6 @@ function deactivate_divs(blocks::Vector{OCBlock})::Vector{OCBlock}
 end
 
 
-"""
-$(SIGNATURES)
-
-Given a candidate header block, check that the opening `#` is at the start of a line, otherwise
-ignore the block.
-"""
-function validate_header_block(β::OCBlock)::Bool
-    # skip non-header blocks
-    β.name ∈ MD_HEADER || return true
-    # if it's a header block, have a look at the opening token
-    τ = otok(β)
-    # check if it overlaps with the first character
-    from(τ) == 1 && return true
-    # otherwise check if the previous character is a linereturn
-    s = str(β.ss) # does not allocate
-    prevc = s[prevind(str(β.ss), from(τ))]
-    prevc == '\n' && return true
-    return false
-end
-
-
 """
 $(SIGNATURES)
 

src/jd_vars.jl

@@ -49,12 +49,14 @@ is processed.
     LOCAL_PAGE_VARS["hasmath"]  = Pair(true,    (Bool,))
     LOCAL_PAGE_VARS["hascode"]  = Pair(false,   (Bool,))
     LOCAL_PAGE_VARS["date"]     = Pair(Date(1), (String, Date, Nothing))
-    LOCAL_PAGE_VARS["jd_ctime"] = Pair(Date(1), (Date,))   # time of creation
-    LOCAL_PAGE_VARS["jd_mtime"] = Pair(Date(1), (Date,))   # time of last modification
-    LOCAL_PAGE_VARS["jd_rpath"] = Pair("",      (String,)) # local path to file src/[...]/blah.md
     LOCAL_PAGE_VARS["lang"]     = Pair("julia", (String,)) # default lang for indented code
     LOCAL_PAGE_VARS["reflinks"] = Pair(true,    (Bool,))   # whether there are reflinks or not
 
+    # page vars used by judoc, should not be accessed or defined
+    LOCAL_PAGE_VARS["jd_ctime"]  = Pair(Date(1), (Date,))   # time of creation
+    LOCAL_PAGE_VARS["jd_mtime"]  = Pair(Date(1), (Date,))   # time of last modification
+    LOCAL_PAGE_VARS["jd_rpath"]  = Pair("",      (String,)) # local path to file src/[...]/blah.md
+
     # If there are GLOBAL vars that are defined, they take precedence
     local_keys   = keys(LOCAL_PAGE_VARS)
     for k in keys(GLOBAL_PAGE_VARS)
@@ -73,16 +75,51 @@ the title, the refstring version of the title, the occurence number and the leve
 """
 const PAGE_HEADERS = Dict{Int,Tuple{AS,AS,Int,Int}}()
 
-
 """
 $(SIGNATURES)
+
+Empties `PAGE_HEADERS`.
 """
 @inline function def_PAGE_HEADERS!()::Nothing
     empty!(PAGE_HEADERS)
     return nothing
 end
 
 
+"""
+PAGE_FNREFS
+
+Keep track of name of seen footnotes; the order is kept as it's a list.
+"""
+const PAGE_FNREFS = String[]
+
+"""
+$(SIGNATURES)
+
+Empties `PAGE_FNREFS`.
+"""
+@inline function def_PAGE_FNREFS!()::Nothing
+    empty!(PAGE_FNREFS)
+    return nothing
+end
+
+"""
+PAGE_LINK_DEFS
+
+Keep track of link def candidates
+"""
+const PAGE_LINK_DEFS = LittleDict{String,String}()
+
+"""
+$(SIGNATURES)
+
+Empties `PAGE_LINK_DEFS`.
+"""
+@inline function def_PAGE_LINK_DEFS!()::Nothing
+    empty!(PAGE_LINK_DEFS)
+    return nothing
+end
+
 """
 GLOBAL_LXDEFS
 

src/misc_html.jl

@@ -12,6 +12,13 @@ Convenience function for a list item
 """
 html_li(in::AS) = "<li>$(in)</li>"
 
+"""
+$SIGNATURES
+
+Convenience function for a sup item
+"""
+html_sup(id::String, in::AS) =  "<sup id=\"$id\">$in</sup>"
+
 """
 $(SIGNATURES)
 
@@ -25,9 +32,10 @@ $(SIGNATURES)
 Convenience function to introduce a hyper reference.
 """
 function html_ahref(link::AS, name::Union{Int,AS};
-                    title::AS="")
-    a  = "<a href=\"$link\""
+                    title::AS="", class::AS="")
+    a  = "<a href=\"$(htmlesc(link))\""
     a *= attr(:title, title)
+    a *= attr(:class, class)
     a *= ">$name</a>"
     a
 end