Make PackageCompiler compulsory; use FunctionTestData throughout; bui…

…ld special exception into Runtime; remove some options from create_local_image; add build_at_path parameter for better debugging

Project.toml

@@ -1,7 +1,7 @@
 name = "Jot"
 uuid = "1043c0aa-ccf1-46a7-999a-bd3d0ee21baa"
 authors = ["Chris Harris <[email protected]> and contributors"]
-version = "0.1.0"
+version = "0.5.0"
 
 [deps]
 Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0"
@@ -26,7 +26,7 @@ PackageCompiler = "2"
 Parameters = "0.12"
 PrettyTables = "1"
 StructTypes = "1"
-julia = "1.3"
+julia = "1.7"
 
 [extras]
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

docs/Project.toml

@@ -1,5 +1,6 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+Jot = "1043c0aa-ccf1-46a7-999a-bd3d0ee21baa"
 
 [compat]
 Documenter = "0.27"

docs/src/Examples.md

@@ -15,14 +15,16 @@ ex1_lambda = create_lambda_function(ex1_remote_image)
 ex2_responder = get_responder("/path/to/script.jl", :response_func, Dict; dependencies=["SpecialFunctions"])
 ex2_local_image = create_local_image("ex2", ex2_responder)
 ex2_remote_image = push_to_ecr!(ex2_local_image)
-ex1_lambda = create_lambda_function(ex2_remote_image)
+ex2_lambda = create_lambda_function(ex2_remote_image)
 ```
 
-### To make a package into a LocalImage, using PackageCompiler to reduce its cold start time...
-... where the package root (containing the Project.toml) is `/path/to/project`, and the package contains a function called `response_func`, that takes a single argument of type `Vector{T} where {T <: Number}`:
+### To make a package into a local docker image, and test it...
+... where the package root (containing the Project.toml) is `/path/to/project`, and the package contains a function called `response_func`, that takes a single argument of type `String` and appends " Responded" to the end of it:
 ```
-ex3_responder = get_responder("/path/to/project", :response_func, Vector)
-ex3_local_image = create_local_image("ex3", ex3_responder; package_compile=true)
+ex3_responder = get_responder("/path/to/project", :response_func, String)
+test_data = FunctionTestData("test", "test Responded")
+ex3_local_image = create_local_image("ex3", ex3_responder; function_test_data=test_data)
+run_local_image_test(ex3_local_image, test_data)
 ```
 
 ### To make a package on github into a responder...
@@ -39,14 +41,6 @@ using IntVectorResponder
 ex5_responder = get_responder(IntVectorResponder, :response_func, Vector{Int64})
 ```
 
-### To make a package into a local docker image, and test it...
-... where the package root (containing the Project.toml) is `/path/to/project`, and the package contains a function called `response_func`, that takes a single argument of type `String` and appends " Responded" to the end of it:
-```
-ex6_responder = get_responder("/path/to/project", :response_func, String)
-ex6_local_image = create_local_image("ex6", ex6_responder)
-run_local_image_test(ex6_local_image, "test", "test Responded")
-```
-
 ### To see if a local docker image has the same function as a remote image...
 ... where local_image is a local docker image, and remote_image is hosted on AWS ECR; the matches function checks that the underlying code of the local image and the remote image match:
 ```

docs/src/Function_Performance.md

@@ -1,30 +1,29 @@
 # Function Performance
 
-### Naive vs Precompiled vs PackageCompiled
-Depending on how the `create_local_image` function is called, the resulting lambda function will be in one of three possible states:
-- If you have called `create_local_image` without either the `function_test_data` or `package_compile` parameters, your function will have been neither precompiled nor PackageCopmiled. Of the three states, this is the slowest, with invocations of the function taking the maximum possible time. This is fine for testing but not for production use.
-- If you have called `create_local_image` with `function_test_data`, but not `package_compile`, your function will be precompiled, but not PackageCompiled. Precompilation is a Julia-native concept, and it means that any compilation required by the function has been done in advance, and stored as part of the docker image.
-- If you have called `create_local_image` with `function_test_data` and with `package_compile=true`, your function will be both precompiled, and PackageCompiled. PackageCompiled means that [PackageCompiler.jl](https://github.com/JuliaLang/PackageCompiler.jl) has been used to create a Julia [system image](https://docs.julialang.org/en/v1/devdocs/sysimg/), and that this system image is part of the docker image. This will result in very fast Lambda function run times, and is highly recommended for production use.
+### The role of PackageCompiler.jl
+Jot uses [PackageCompiler.jl](https://github.com/JuliaLang/PackageCompiler.jl) to optimize the performance of its generated functions. The package compilation process uses the concept of an "exemplar session" to discover which Julia methods need to be compiled. This "exemplar session" will, ideally, call all methods that will be used by the actual docker image, when running on AWS Lambda. With Jot, an exemplar session is created automatically during the `create_local_image` stage. This test run simulates how AWS Lambda will invoke the responder function. There are two variables that affect the contents of this test run, and consequently the ultimate performance of the lambda function:
 
-When setting the `package_compile` option to `true`, you will need to also pass a `FunctionTestData` object to the `function_test_data` parameter of `create_local_image`. This defines a sample argument to pass when testing your lambda function, and the expected response that the lambda function should return when passed that argument.
+- If you have called `create_local_image` with the `function_test_data` parameter, then the responder function will be called with the `test_arument` parameter supplied in the `function_test_data`. If `function_test_data` is not passed, then the Jot runtime on the docker image - used for HTTP communication with AWS, and JSON reading/writing - will still be part of the exemplar session, but the responder function will not.
+- If your responder is a package with a test suite, and you have called `create_local_image` with `run_tests_during_package_compile=true`, then this test suite will be executed as part of the exemplar session. Since the `test_argument` parameter can only invoke a single code path, having a test suite (which presumably tests multiple code paths) is a more robust way to improve performance.
 
-So if your responder function takes a vector of integers, and increases each element by 1:
+### Passing function_test_data
+If your responder function takes a vector of integers, and increases each element by 1:
 ```
 open("increment_vector.jl", "w") do f
   write(f, "increment_vector(v::Vector{Int}) = map(x -> x + 1, v)")
 end
 increment_responder = get_responder("./increment_vector.jl", :increment_vector, Vector{Int})
 ```
 
-... then your `FunctionTestData` might look like this:
+... then your `function_test_data` might look like this:
 ```
 function_test_data = FunctionTestData([1,2,3], [2,3,4])
 ```
 where `[1,2,3]` is the argument you intend to pass to the responder, and `[2,3,4]` is the response you are expecting.
 
 ... and your call to `create_local_image` might look like this:
 ```
-`create_local_image(increment_responder; function_test_data=function_test_data, package_compile=true)`
+`create_local_image(increment_responder; function_test_data=function_test_data)`
 ```
 
 ### Hot vs warm vs cold starts

docs/src/Functions.md

@@ -20,34 +20,32 @@ create_lambda_function(
     memory_size::Int64 = 2000,
   )
 create_local_image(
-    responder::AbstractResponder;
+    responder::Responder;
     image_suffix::Union{Nothing, String} = nothing,
     aws_config::Union{Nothing, AWSConfig} = nothing,
     image_tag::String = "latest",
     no_cache::Bool = false,
     julia_base_version::String = "1.8.4",
     julia_cpu_target::String = "x86-64",
     function_test_data::Union{Nothing, FunctionTestData} = nothing,
-    package_compile::Bool = false,
     user_defined_labels::AbstractDict{String, String} = OrderedDict{String, String}(),
     dockerfile_update::Function = x -> x,
     build_args::AbstractDict{String, String} = OrderedDict{String, String}(),
+    run_tests_during_package_compile::Bool = false,
   )
 delete!(con::Container)
 delete!(repo::ECRRepo)
 delete!(func::LambdaFunction)
 delete!(image::LocalImage; force::Bool=false)
 get_dockerfile(
-    responder::AbstractResponder,
-    julia_base_version::String;
+    responder::Responder,
     user_defined_labels::AbstractDict{String, String} = AbstractDict{String, String}(),
     dockerfile_update::Function = x -> x,
-    package_compile::Bool,
   )
 get_ecr_repo(image::LocalImage)
 get_ecr_repo(repo_name::String)
 create_lambda_components(
-    res::AbstractResponder;
+    res::Responder;
     image_suffix::Union{Nothing, String} = nothing,
     aws_config::Union{Nothing, AWSConfig} = nothing,
     image_tag::String = "latest",
@@ -72,16 +70,16 @@ get_remote_image(lambda_function::LambdaFunction)
 get_remote_image(local_image::LocalImage)
 get_remote_image(identity::AbstractString)
 get_responder(
-    path_url::String,
+    mod::Module,
     response_function::Symbol,
     response_function_param_type::Type;
-    dependencies = Vector{String}(),
-    registry_urls = Vector{String}(),
+    registry_urls::Vector{<:AbstractString} = Vector{String}(),
   )
 get_responder(
-    mod::Module,
+    path_url::String,
     response_function::Symbol,
     response_function_param_type::Type;
+    dependencies::Vector{String} = Vector{String}(),
     registry_urls::Vector{String} = Vector{String}(),
   )
 get_user_labels(l::Union{LocalImage, ECRRepo, RemoteImage, LambdaFunction})
@@ -100,19 +98,17 @@ push_to_ecr!(image::LocalImage)
 run_image_locally(local_image::LocalImage; detached::Bool=true)
 run_local_image_test(
     image::LocalImage,
-    function_argument::Any = "",
-    expected_response::Any = nothing;
+    function_test_data::Union{Nothing, FunctionTestData};
     then_stop::Bool = false,
   )
 run_lambda_function_test(
     func::LambdaFunction,
-    function_test_data::FunctionTestData;
+    function_test_data::Union{Nothing, FunctionTestData};
     check_function_state::Bool = false,
   )
 run_test(
-    l::LambdaComponents;
-    function_argument::Any = "",
-    expected_response::Any = nothing,
+    l::LambdaComponents,
+    function_test_data::Union{Nothing, FunctionTestData} = nothing,
   )
 send_local_request(request::Any; local_port::Int64 = 9000)
 show_lambdas()

docs/src/Guide.md

@@ -15,16 +15,16 @@ Jot uses a four-step process to go from a given block of Julia code, to a workin
 
 A `Responder` is our chosen code for responding to Lambda Function calls. It can be as simple as a single function in a short Julia script, or it can be a fully-developed package with multiple dependencies.
 
-A `LocalImage` is a locally-hosted docker image that has Julia installed. In the process of creating a `LocalImage`, the code specified by the `Responder` will be added to it and enabled for use. A `LocalImage` is unique by both - in Docker terminology - its *Repository* (basic identity) and its *Tag* (version). Therefore different versions/*tags* of the same basic image will be represented by different `LocalImage`s. As well as the code specified by the `Responder`, a given `LocalImage` will also have the `Jot.jl` package itself installed. `Jot.jl` hosts the `Responder`, handling HTTP routing and JSON conversion. Additionally, the `LocalImage` has *AWS RIE* installed, a utility provided by AWS that emulates the Lambda run-time environment and so enables local testing of the function.
+A `LocalImage` is a locally-hosted docker image that has Julia installed. In the process of creating a `LocalImage`, the code specified by the `Responder` will be added to it and enabled for use. A `LocalImage` is unique by both - in Docker terminology - its *Repository* (basic identity) and its *Tag* (version). Therefore different versions/*tags* of the same basic image will be represented by different `LocalImage`s. Behind-the-scenes, as well as the code specified by the `Responder`, `Jot.jl` will also be added to a `LocalImage`. Then, within the `LocalImage`, `Jot.jl` hosts the `Responder`, handling HTTP routing and JSON conversion. Additionally, the `LocalImage` has *AWS RIE* installed, a utility provided by AWS that emulates the Lambda run-time environment and so enables local testing of the function.
 
 A `RemoteImage` represents a `LocalImage`, after it has been uploaded to [AWS ECR](https://aws.amazon.com/ecr/). All `RemoteImage`s must be stored in an ECR Repo. This repo maps to the image *Repository*. A repo may therefore contain multiple versions/*tags* for the same *Repository*; therefore multiple `RemoteImage`s that share a *Repository* may be stored in the same ECR Repo.
 
 A `LambdaFunction` is the final stage in the process and represents a working Lambda function, powered by a single `RemoteImage`.
 
-## Best practices
+## PackageCompiler.jl is used to improve performance
+[PackageCompiler.jl](https://github.com/JuliaLang/PackageCompiler.jl) is a Julia package that pre-compiles Julia methods. This eliminates almost all of the usual delay while Julia starts up, and so reduces Lambda Function [cold start-up times](https://aws.amazon.com/blogs/compute/operating-lambda-performance-optimization-part-1/) by around 75%, making it competitive with any other language used for AWS Lambda. Passing `function_test_data` to the `create_local_image` function further improves performance, as well as allowing Jot to test that the generated function is working correctly. See [function performance](Function_Performance.md) for more details.
 
-### Using PackageCompiler.jl
-[PackageCompiler.jl](https://github.com/JuliaLang/PackageCompiler.jl) is a Julia package that pre-compiles methods. This can be done during the image creation process, and the `create_local_image` function has a `package_compile` option (default `false`) to indicate whether this should be used. Setting `package_compile` to `true` is **highly recommended** for production use; it eliminates almost all of the usual delay while Julia starts up, and so reduces Lambda Function [cold start-up times](https://aws.amazon.com/blogs/compute/operating-lambda-performance-optimization-part-1/) by around 75%, making it competitive with any other language used for AWS Lambda. See [function performance](Function_Performance.md) for more details.
+## Best practices
 
 ### Working around the one-function-per-container limit
 The Lambda API limits you to one function per container. In practice, dividing up all your functions into different containers is not practical. Instead, have the responding function expect a Dict, then use one of the fields of the dict to indicate the function that should be called. The responding function can then just forward the other parameters to the appropriate function.

docs/src/Types.md

@@ -1,15 +1,14 @@
 # Types
 
 ```@docs
-AbstractResponder
 AWSConfig
 Container
 FunctionTestData
 InvocationTimeBreakdown
 LambdaFunction
 LambdaFunctionInvocationLog
 LocalImage
-LocalPackageResponder
+Responder
 LogEvent
 RemoteImage
 ```

docs/src/index.md

@@ -46,7 +46,7 @@ lambda_components |> with_remote_image! |> with_lambda_function! |> run_test
 
 ## Package Features
 - Easily create AWS Lambda functions from Julia packages or scripts
-- PackageCompiler.jl may be optionally used to greatly speed up cold start times
+- PackageCompiler.jl is used to maximize performance and minimize cold start times
 - JSON read/write and error handling is handled by Jot - you just write standard Julia
 - Allows easy checking for version consistency - ie, is a given Lambda Function using the correct code?
 

result

@@ -0,0 +1 @@
+/nix/store/s859cbq2airmc7k0sca13bnmx39ccp12-helix-23.03

src/BuildDockerfile.jl

@@ -29,17 +29,14 @@ function dockerfile_add_utilities()::String
 end
 
 function dockerfile_add_runtime_directories(
-    julia_depot_path::String,
-    temp_path::String,
     runtime_path::String
   )::String
   """
-  RUN mkdir -p $julia_depot_path
-  ENV JULIA_DEPOT_PATH=$julia_depot_path
   RUN mkdir -p $temp_path
   ENV TMPDIR=$temp_path
   RUN mkdir -p $runtime_path
   WORKDIR $runtime_path
+  ENV JULIA_DEPOT_PATH="$temp_path:$runtime_path/$julia_depot_dir_name"
   """
 end
 
@@ -49,34 +46,30 @@ function dockerfile_copy_build_dir()::String
   """
 end
 
+function dockerfile_move_depot_path_to_tmp()::String
+  """
+  RUN mv ./$julia_depot_dir_name /tmp/
+  RUN echo "\$(ls /tmp/$julia_depot_dir_name)"
+  """
+end
+
 function dockerfile_add_responder(
     runtime_path::String,
-    res::LocalPackageResponder,
+    res::Responder,
   )::String
   using_pkg_script = "using Pkg; "
-  add_module_script = "Pkg.develop(PackageSpec(path=\\\"$runtime_path/$(res.package_name)\\\")); "
+  package_name = get_package_name(res)
+  add_module_script = "Pkg.develop(PackageSpec(path=\\\"$runtime_path/$package_name\\\")); "
   instantiate_script = "Pkg.instantiate(); "
   """
   RUN julia -e \"$using_pkg_script$add_module_script$instantiate_script\"
   """
 end
 
-function dockerfile_add_environment(
-    responder::LocalPackageResponder,
+function dockerfile_create_julia_environment(
   )::String
-  test_running = get(ENV, "JOT_TEST_RUNNING", nothing)
-  jot_branch = if isnothing(test_running) || test_running == "false"
-    "main"
-  else
-    readchomp(`git branch --show-current`)
-  end
-  responder_package_path = "./$(responder.package_name)"
-  create_env_script = get_create_julia_environment_script(
-    responder_package_path, "."; jot_branch
-  )
-  create_env_statement = replace(create_env_script, "\n" => "; ") |> nest_quotes
   """
-  RUN julia -e \"$create_env_statement\"
+  RUN sh create_environment
   """
 end
 
@@ -143,7 +136,6 @@ function dockerfile_add_labels(labels::Labels)::String
 end
 
 function get_dockerfile_build_cmd(
-    dockerfile::String,
     image_full_name_plus_tag::String,
     no_cache::Bool,
     build_args::AbstractDict{String, String},