From 095968b767a35dd4d34452c759194281587b9e5c Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Thu, 1 Feb 2024 11:41:57 +1300 Subject: [PATCH] Initial commit with a PoC --- .github/dependabot.yml | 7 ++ .github/workflows/CI.yml | 41 +++++++++ .github/workflows/CompatHelper.yml | 16 ++++ .github/workflows/TagBot.yml | 31 +++++++ .gitignore | 1 + CHANGELOG.md | 8 ++ LICENSE | 22 +++++ Project.toml | 16 ++++ README.md | 16 ++++ src/CodeEvaluation.jl | 7 ++ src/namedsandboxes.jl | 22 +++++ src/sandbox.jl | 135 +++++++++++++++++++++++++++++ test/runtests.jl | 85 ++++++++++++++++++ 13 files changed, 407 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/CI.yml create mode 100644 .github/workflows/CompatHelper.yml create mode 100644 .github/workflows/TagBot.yml create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 LICENSE create mode 100644 Project.toml create mode 100644 README.md create mode 100644 src/CodeEvaluation.jl create mode 100644 src/namedsandboxes.jl create mode 100644 src/sandbox.jl create mode 100644 test/runtests.jl diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..700707c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..054444b --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,41 @@ +name: CI +on: + push: + branches: + - main + tags: ['*'] + pull_request: + workflow_dispatch: +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + permissions: # needed to allow julia-actions/cache to proactively delete old caches that it has created + actions: write + contents: read + strategy: + fail-fast: false + matrix: + version: + - '1.10' + - '1.6' + - 'nightly' + os: + - ubuntu-latest + arch: + - x64 + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v1 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 0000000..cba9134 --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,16 @@ +name: CompatHelper +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} + run: julia -e 'using CompatHelper; CompatHelper.main()' diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml new file mode 100644 index 0000000..2bacdb8 --- /dev/null +++ b/.github/workflows/TagBot.yml @@ -0,0 +1,31 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: + inputs: + lookback: + default: 3 +permissions: + actions: read + checks: read + contents: write + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b067edd --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/Manifest.toml diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ac9111c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +# Release notes + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Version [v0.1.0] - 2024-??-?? + +Initial release. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fa86886 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2016-2021: Michael Hatherly, Morten Piibeleht, Fredrik Ekre, and Documenter.jl contributors +Copyright (c) 2024 Morten Piibeleht and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..82ddbf7 --- /dev/null +++ b/Project.toml @@ -0,0 +1,16 @@ +name = "CodeEvaluation" +uuid = "5a076611-96cb-4f02-9d3a-9e309f06f8ff" +authors = ["Morten Piibeleht and contributors"] +version = "0.0.1" + +[deps] +IOCapture = "b5f81e59-6552-4d32-b1f0-c071b021bf89" + +[compat] +julia = "1.6" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..a25bbdd --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# CodeEvaluation + +[![Build Status](https://github.com/JuliaDocs/CodeEvaluation.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/JuliaDocs/CodeEvaluation.jl/actions/workflows/CI.yml?query=branch%3Amain) +[![PkgEval](https://JuliaCI.github.io/NanosoldierReports/pkgeval_badges/C/CodeEvaluation.svg)](https://JuliaCI.github.io/NanosoldierReports/pkgeval_badges/C/CodeEvaluation.html) + +> [!NOTE] +> This package is in active development, and not yet registered. + +A small utility package to emulate executing Julia code in a clean `Main` module. + +The package uses [IOCapture.jl](https://github.com/JuliaDocs/IOCapture.jl) to perform output capture of the evaluated code. + +> [!WARNING] +> The code evaluation is not thread-safe. +> This is because for each evaluation, the code has to change the Julia processe's working directory with `cd`. +> This global change will also affect any code running in parallel in other tasks or threads. diff --git a/src/CodeEvaluation.jl b/src/CodeEvaluation.jl new file mode 100644 index 0000000..decd2a9 --- /dev/null +++ b/src/CodeEvaluation.jl @@ -0,0 +1,7 @@ +module CodeEvaluation +import IOCapture + +include("sandbox.jl") +include("namedsandboxes.jl") + +end diff --git a/src/namedsandboxes.jl b/src/namedsandboxes.jl new file mode 100644 index 0000000..70262ba --- /dev/null +++ b/src/namedsandboxes.jl @@ -0,0 +1,22 @@ +struct NamedSandboxes + _pwd::String + _prefix::String + _sandboxes::Dict{Symbol,Sandbox} + + function NamedSandboxes(pwd::AbstractString, prefix::AbstractString = "") + unique_prefix = _gensym_string() + prefix = isempty(prefix) ? unique_prefix : string(prefix, "_", unique_prefix) + return new(pwd, prefix, Dict{Symbol,Sandbox}()) + end +end + +function Base.get!(s::NamedSandboxes, name::Union{AbstractString,Nothing} = nothing) + sym = if isnothing(name) || isempty(name) + Symbol("__", s._prefix, "__", _gensym_string()) + else + Symbol("__", s._prefix, "__named__", name) + end + # Either fetch and return an existing sandbox from the meta dictionary (based on the generated name), + # or initialize a new clean one, which gets stored in meta for future re-use. + return get!(() -> Sandbox(sym, s._pwd), s._sandboxes, sym) +end diff --git a/src/sandbox.jl b/src/sandbox.jl new file mode 100644 index 0000000..00b48ea --- /dev/null +++ b/src/sandbox.jl @@ -0,0 +1,135 @@ +# Constructs a new sandbox module, that emulates an emptry Julia Main module. +function _sandbox_module(sym::Symbol) + # If the module does not exists already, we need to construct a new one. + m = Module(sym) + # eval(expr) is available in the REPL (i.e. Main) so we emulate that for the sandbox + Core.eval(m, :(eval(x) = Core.eval($m, x))) + # modules created with Module() does not have include defined + Core.eval(m, :(include(x) = Base.include($m, abspath(x)))) + return m +end + +# TODO: add a method to write to _codebuffer without evaluating the code +# This is to enable the "ContinuedCode" use case, where you want to "prepare" +# the input code over multiple writes. +mutable struct Sandbox + m::Module + pwd::String + _codebuffer::IOBuffer + Sandbox(sym::Symbol, pwd::AbstractString) = new(_sandbox_module(sym), pwd, IOBuffer()) +end + +# TODO: by stripping the #-s, we're probably losing the uniqueness guarantee? +_gensym_string() = lstrip(string(gensym()), '#') + +function evaluate!(sandbox::Sandbox; ansicolor::Bool=true) + code = String(take!(sandbox._codebuffer)) + + # Evaluate the code block. We redirect stdout/stderr to `buffer`. + result, buffer = nothing, IOBuffer() + + # TODO: use keywords, linenumbernode? + @show parseblock(code) + for (ex, str) in parseblock(code) + c = IOCapture.capture(rethrow = InterruptException, color = ansicolor) do + cd(sandbox.pwd) do + Core.eval(sandbox.m, ex) + end + end + Core.eval(sandbox.m, Expr(:global, Expr(:(=), :ans, QuoteNode(c.value)))) + result = c.value + print(buffer, c.output) + if c.error + #bt = Documenter.remove_common_backtrace(c.backtrace) + bt = c.backtrace + @error """ + Error executing code: + ``` + $(code) + ``` + """ exception = (c.value, bt) + return + end + end + + return (; result, output = String(take!(buffer))) +end + +Base.write(sandbox::Sandbox, data) = write(sandbox._codebuffer, data) + +""" +Returns a vector of parsed expressions and their corresponding raw strings. + +Returns a `Vector` of tuples `(expr, code)`, where `expr` is the corresponding expression +(e.g. a `Expr` or `Symbol` object) and `code` is the string of code the expression was +parsed from. + +The keyword argument `skip = N` drops the leading `N` lines from the input string. + +If `raise=false` is passed, the `Meta.parse` does not raise an exception on parse errors, +but instead returns an expression that will raise an error when evaluated. `parseblock` +returns this expression normally and it must be handled appropriately by the caller. + +The `linenumbernode` can be passed as a `LineNumberNode` to give information about filename +and starting line number of the block (requires Julia 1.6 or higher). +""" +function parseblock(code::AbstractString; skip = 0, keywords = true, raise=true, linenumbernode=nothing) + # Drop `skip` leading lines from the code block. Needed for deprecated `{docs}` syntax. + code = string(code, '\n') + code = last(split(code, '\n', limit = skip + 1)) + endofstr = lastindex(code) + results = [] + cursor = 1 + while cursor < endofstr + # Check for keywords first since they will throw parse errors if we `parse` them. + line = match(r"^(.*)\r?\n"m, SubString(code, cursor)).match + keyword = Symbol(strip(line)) + (ex, ncursor) = if keywords && haskey(Docs.keywords, keyword) + (QuoteNode(keyword), cursor + lastindex(line)) + else + try + Meta.parse(code, cursor; raise=raise) + catch err + @error "parse error" + break + end + end + str = SubString(code, cursor, prevind(code, ncursor)) + if !isempty(strip(str)) && ex !== nothing + push!(results, (ex, str)) + end + cursor = ncursor + end + if linenumbernode isa LineNumberNode + exs = Meta.parseall(code; filename=linenumbernode.file).args + @assert length(exs) == 2 * length(results) "Issue at $linenumbernode:\n$code" + for (i, ex) in enumerate(Iterators.partition(exs, 2)) + @assert ex[1] isa LineNumberNode + expr = Expr(:toplevel, ex...) # LineNumberNode + expression + # in the REPL each evaluation is considered a new file, e.g. + # REPL[1], REPL[2], ..., so try to mimic that by incrementing + # the counter for each sub-expression in this code block + if linenumbernode.file === Symbol("REPL") + newfile = "REPL[$i]" + # to reset the line counter for each new "file" + lineshift = 1 - ex[1].line + update_linenumbernodes!(expr, newfile, lineshift) + else + update_linenumbernodes!(expr, linenumbernode.file, linenumbernode.line) + end + results[i] = (expr , results[i][2]) + end + end + results +end + +function update_linenumbernodes!(x::Expr, newfile, lineshift) + for i in 1:length(x.args) + x.args[i] = update_linenumbernodes!(x.args[i], newfile, lineshift) + end + return x +end +update_linenumbernodes!(x::Any, newfile, lineshift) = x +function update_linenumbernodes!(x::LineNumberNode, newfile, lineshift) + return LineNumberNode(x.line + lineshift, newfile) +end diff --git a/test/runtests.jl b/test/runtests.jl new file mode 100644 index 0000000..6594da0 --- /dev/null +++ b/test/runtests.jl @@ -0,0 +1,85 @@ +using CodeEvaluation +using Test + +@testset "CodeEvaluation.jl" begin + @testset "parseblock" begin + code = """ + x += 3 + γγγ_γγγ + γγγ + """ + exprs = CodeEvaluation.parseblock(code) + + @test isa(exprs, Vector) + @test length(exprs) === 3 + + @test isa(exprs[1][1], Expr) + @test exprs[1][1].head === :+= + @test exprs[1][2] == "x += 3\n" + + @test exprs[2][2] == "γγγ_γγγ\n" + + @test exprs[3][1] === :γγγ + if VERSION >= v"1.10.0-DEV.1520" # JuliaSyntax merge + @test exprs[3][2] == "γγγ\n\n" + else + @test exprs[3][2] == "γγγ\n" + end + end + + # These tests were covering cases reported in + # https://github.com/JuliaDocs/Documenter.jl/issues/749 + # https://github.com/JuliaDocs/Documenter.jl/issues/790 + # https://github.com/JuliaDocs/Documenter.jl/issues/823 + let parse(x) = CodeEvaluation.parseblock(x) + for LE in ("\r\n", "\n") + l1, l2 = parse("x = Int[]$(LE)$(LE)push!(x, 1)$(LE)") + @test l1[1] == :(x = Int[]) + @test l2[1] == :(push!(x, 1)) + if VERSION >= v"1.10.0-DEV.1520" # JuliaSyntax merge + @test l1[2] == "x = Int[]$(LE)$(LE)" + @test l2[2] == "push!(x, 1)$(LE)\n" + else + @test l1[2] == "x = Int[]$(LE)" + @test l2[2] == "push!(x, 1)$(LE)" + end + end + end + + @testset "NamedSandboxes" begin + sandboxes = CodeEvaluation.NamedSandboxes(@__DIR__, "testsandbox") + sb1 = get!(sandboxes, "foo") + sb2 = get!(sandboxes, "bar") + sb3 = get!(sandboxes, "foo") + @test sb1.m !== sb2.m + @test sb1.m === sb3.m + @test sb2.m !== sb3.m + end + + @testset "evaluate!" begin + let sb = CodeEvaluation.Sandbox(:foo, @__DIR__) + write(sb, "2 + 2") + (result, output) = CodeEvaluation.evaluate!(sb) + @test result === 4 + @test output === "" + end + + let sb = CodeEvaluation.Sandbox(:foo, @__DIR__) + write(sb, "print(\"123\")") + (result, output) = CodeEvaluation.evaluate!(sb) + @test result === nothing + @test output === "123" + end + + let sb = CodeEvaluation.Sandbox(:foo, @__DIR__) + write(sb, """ + x = 2 + 2 + print(x) + x + 1 + """) + (result, output) = CodeEvaluation.evaluate!(sb) + @test result === 5 + @test output === "4" + end + end +end