Merge pull request #39 from sstroemer/fix-38

fix: proper parsing of string expressions that contain `foo.bar:value`
ait-energy · Dec 10, 2024 · b3d8d88 · b3d8d88
2 parents 5a798e6 + 3fdc44f
commit b3d8d88
Show file tree

Hide file tree

Showing 8 changed files with 150 additions and 44 deletions.
diff --git a/src/core/expression.jl b/src/core/expression.jl
@@ -30,42 +30,80 @@ function _string_to_fevalexpr(model::JuMP.Model, @nospecialize(str::AbstractStri
                 separator = popfirst!(tokens)   # TODO: this is actually just `next_token`
                 next_token = popfirst!(tokens)  # this is the token after the separator
 
-                push!(access_order, join(JuliaSyntax.untokenize.((token, separator, next_token), str)))
+                push!(access_order, join(JuliaSyntax.untokenize.((token, separator, next_token), str)))  # TODO: the join is waste, untokenize everything at once
                 write(buf, "__el[$(length(access_order) - starting_access_index + 1)]")
                 last_token = next_token
-            else
-                # This may be a "carrier" as part of a conversion expression, or ...
-                # we may know that this is NOT a conversion expression.
-                elem = JuliaSyntax.untokenize(token, str)
-
-                if is_conv_expr && haskey(internal(model).model.carriers::Dict{String, IESopt.Carrier}, elem)
-                    # A conversion expression (nice) AND it is a valid carrier.
-                    e = JuliaSyntax.parsestmt(Expr, String(take!(buf)))
-
-                    if length(access_order) >= starting_access_index
-                        f = @RuntimeGeneratedFunction(:(function (__el::Vector{Union{Float64, String}})
-                            return $e
-                        end))
-                        push!(
-                            extracted_expressions,
-                            (name=elem, func=f, elements=access_order[starting_access_index:end]),
-                        )
-                        starting_access_index = length(access_order) + 1
+
+                continue
+            end
+
+            if !isempty(tokens) && (JuliaSyntax.kind(first(tokens)) == JuliaSyntax.K".")
+                # This might be something like `foobar.electricity_supply.size:value`, so we need to peek ahead.
+                if length(tokens) < 2
+                    @critical "Failed to parse string to valid Julia syntax, since it seems to end with a `.`?" input =
+                        str token = JuliaSyntax.untokenize(token, str)
+                end
+
+                continue_outer_loop = true
+                for i in eachindex(tokens)[2:end]
+                    next_token = tokens[i]
+
+                    JuliaSyntax.kind(next_token) == JuliaSyntax.K"Identifier" && continue
+                    JuliaSyntax.kind(next_token) == JuliaSyntax.K"." && continue
+
+                    if JuliaSyntax.kind(next_token) == JuliaSyntax.K":"
+                        # Ok, we found an access to a Decision after some other stuff.
+                        #   => Handle it and skip the code below.
+
+                        if (length(tokens) < (i + 1)) || JuliaSyntax.kind(tokens[i + 1]) != JuliaSyntax.K"Identifier"
+                            @critical "Failed to parse string to valid Julia syntax" input = str token =
+                                JuliaSyntax.untokenize(token, str)
+                        end
+
+                        push!(access_order, join(JuliaSyntax.untokenize.((token, tokens[1:(i + 1)]...), str)))  # TODO: the join is waste, untokenize everything at once
+                        write(buf, "__el[$(length(access_order) - starting_access_index + 1)]")
+                        last_token = last([popfirst!(tokens) for _ in 1:(i + 1)])
+
+                        continue_outer_loop = false
+                        break
                     else
-                        value = convert(Float64, eval(e))
-                        push!(extracted_expressions, (name=elem, val=value))
+                        # Ok, something else was found.
+                        #   => Just continue with the code below.
+                        break
                     end
+                end
+
+                continue_outer_loop || continue
+            end
+
+            # This may be a "carrier" as part of a conversion expression, or ...
+            # we may know that this is NOT a conversion expression.
+            elem = JuliaSyntax.untokenize(token, str)
 
-                    last_token = TOK_TOMB
+            if is_conv_expr && haskey(internal(model).model.carriers::Dict{String, IESopt.Carrier}, elem)
+                # A conversion expression (nice) AND it is a valid carrier.
+                e = JuliaSyntax.parsestmt(Expr, String(take!(buf)))
+
+                if length(access_order) >= starting_access_index
+                    f = @RuntimeGeneratedFunction(:(function (__el::Vector{Union{Float64, String}})
+                        return $e
+                    end))
+                    push!(extracted_expressions, (name=elem, func=f, elements=access_order[starting_access_index:end]))
+                    starting_access_index = length(access_order) + 1
                 else
-                    # It is not (either one). Let's hope it's valid Julia syntax (e.g., `sqrt(2)`).
-                    try
-                        # This is the same code as in the "else" below, but chances for errors are higher here ...
-                        write(buf, elem)
-                        last_token = token
-                    catch
-                        @critical "Failed to parse string to valid Julia syntax" input = str token = elem
-                    end
+                    value = convert(Float64, eval(e))
+                    push!(extracted_expressions, (name=elem, val=value))
+                end
+
+                last_token = TOK_TOMB
+            else
+                # It is not (either one). Let's hope it's valid Julia syntax (e.g., `sqrt(2)`).
+                try
+                    # This is the same code as in the "else" below, but chances for errors are higher here ...
+                    write(buf, elem)
+                    last_token = token
+                catch
+                    @critical "Failed to parse string to valid Julia syntax" input = str token = elem
                 end
             end
         else
@@ -75,7 +113,8 @@ function _string_to_fevalexpr(model::JuMP.Model, @nospecialize(str::AbstractStri
     end
 
     if isempty(extracted_expressions)
-        e = JuliaSyntax.parsestmt(Expr, String(take!(buf)))
+        expr_str = String(take!(buf))
+        e = JuliaSyntax.parsestmt(Expr, expr_str)
 
         if length(access_order) >= starting_access_index
             f = @RuntimeGeneratedFunction(:(function (__el::Vector{Union{Float64, String}})

diff --git a/test/Manifest.toml b/test/Manifest.toml
@@ -125,9 +125,9 @@ version = "0.10.38"
 
 [[deps.HiGHS]]
 deps = ["HiGHS_jll", "MathOptInterface", "PrecompileTools", "SparseArrays"]
-git-tree-sha1 = "02fad6652a24cd1356c5dc000c3cca297e13482a"
+git-tree-sha1 = "5360ef81c3952f29624ec75ad0045d079c6379f3"
 uuid = "87dc4568-4c63-4d18-b0c0-bb2238e4078b"
-version = "1.12.1"
+version = "1.12.2"
 
 [[deps.HiGHS_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Zlib_jll"]
@@ -173,9 +173,9 @@ version = "0.21.4"
 
 [[deps.JuMP]]
 deps = ["LinearAlgebra", "MacroTools", "MathOptInterface", "MutableArithmetics", "OrderedCollections", "PrecompileTools", "Printf", "SparseArrays"]
-git-tree-sha1 = "cac0ea273db1c9db803af8ee2921d0a3f5a193a4"
+git-tree-sha1 = "866dd0bf0474f0d5527c2765c71889762ba90a27"
 uuid = "4076af6c-e467-56ae-b986-b466b2749572"
-version = "1.23.4"
+version = "1.23.5"
 
     [deps.JuMP.extensions]
     JuMPDimensionalDataExt = "DimensionalData"
@@ -185,9 +185,9 @@ version = "1.23.4"
 
 [[deps.JuliaInterpreter]]
 deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"]
-git-tree-sha1 = "fc8504eca188aaae4345649ca6105806bc584b70"
+git-tree-sha1 = "10da5154188682e5c0726823c2b5125957ec3778"
 uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a"
-version = "0.9.37"
+version = "0.9.38"
 
 [[deps.LibCURL]]
 deps = ["LibCURL_jll", "MozillaCACerts_jll"]
@@ -245,9 +245,9 @@ version = "1.11.0"
 
 [[deps.LoweredCodeUtils]]
 deps = ["JuliaInterpreter"]
-git-tree-sha1 = "260dc274c1bc2cb839e758588c63d9c8b5e639d1"
+git-tree-sha1 = "688d6d9e098109051ae33d126fcfc88c4ce4a021"
 uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b"
-version = "3.0.5"
+version = "3.1.0"
 
 [[deps.MacroTools]]
 deps = ["Markdown", "Random"]
@@ -312,9 +312,9 @@ uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
 version = "0.5.5+0"
 
 [[deps.OrderedCollections]]
-git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5"
+git-tree-sha1 = "12f1439c4f986bb868acda6ea33ebc78e19b95ad"
 uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-version = "1.6.3"
+version = "1.7.0"
 
 [[deps.Parsers]]
 deps = ["Dates", "PrecompileTools", "UUIDs"]

diff --git a/test/runtests.jl b/test/runtests.jl
@@ -28,5 +28,9 @@ include("src/examples.jl")
         include("src/basic.jl")
     end
 
+    @testset "Fixes (IESopt.jl)" verbose = true begin
+        include("src/issues.jl")
+    end
+
     @run_package_tests verbose = true filter = ti -> (:examples in ti.tags)
 end
diff --git a/test/src/basic.jl b/test/src/basic.jl
@@ -14,9 +14,6 @@
     model = generate!(joinpath(PATH_TESTFILES, "availability_test_failure.iesopt.yaml"))
     @suppress optimize!(model)      # `@test_logs` fails because: https://github.com/JuliaLang/julia/issues/48456
     @test JuMP.termination_status(model) == JuMP.MOI.INFEASIBLE
-
-    model = generate!(joinpath(PATH_TESTFILES, "issue35.iesopt.yaml"))
-    @test access(get_component(model, "electricity_demand").value) ≈ 7e-6
 end
 
 @testset "Filesystem paths" verbose = true begin

diff --git a/test/src/issues.jl b/test/src/issues.jl
@@ -0,0 +1,16 @@
+@testset "#35" begin
+    model = generate!(joinpath(PATH_TESTFILES, "issues", "35", "config.iesopt.yaml"))
+    @test access(get_component(model, "electricity_demand").value) ≈ 7e-6
+end
+
+@testset "#38" begin
+    model = generate!(joinpath(PATH_TESTFILES, "issues", "38", "config.iesopt.yaml"))
+    supplier = get_component(model, "electricity_supply")
+    demand_ub = access(supplier.supply.ub)
+
+    @test demand_ub.terms[supplier.size.var.value] ≈ 2.0
+    @test demand_ub.constant ≈ 3.0
+
+    optimize!(model)
+    @test JuMP.objective_value(model) ≈ 0.0 atol = 0.1
+end
diff --git a/test/test_files/issue35.iesopt.yaml → test/test_files/issues/35/config.iesopt.yaml b/test/test_files/issue35.iesopt.yaml → test/test_files/issues/35/config.iesopt.yaml
diff --git a/test/test_files/issues/38/config.iesopt.yaml b/test/test_files/issues/38/config.iesopt.yaml
@@ -0,0 +1,36 @@
+config:
+  general:
+    version:
+      core: 2.1.0
+    performance:
+      logfile: false
+    verbosity:
+      core: error
+  optimization:
+    problem_type: LP
+    snapshots:
+      count: 1
+    solver:
+      name: highs
+      log: false
+  paths:
+    templates: templates
+
+carriers:
+  electricity: {}
+
+components:
+  electricity_node:
+    type: Node
+    carrier: electricity
+
+  electricity_demand:
+    type: Profile
+    carrier: electricity
+    node_from: electricity_node
+    value: 1
+
+  electricity_supply:
+    type: Supplier
+    node_to: electricity_node
+    cost: 1
diff --git a/test/test_files/issues/38/templates/Supplier.iesopt.template.yaml b/test/test_files/issues/38/templates/Supplier.iesopt.template.yaml
@@ -0,0 +1,14 @@
+parameters:
+  cost: ~
+  node_to: ~
+
+components:
+  supply:
+    type: Profile
+    carrier: electricity
+    mode: ranged
+    ub: 1.0*<self>.size:value + 3.0 + 62.5 * (1.6e-4/0.01 * <self>.size:value)
+    node_to: <node_to>
+  size:
+    type: Decision
+    cost: <cost>