Refactor tests, ensure julia_depot path is writable

Project.toml

@@ -4,7 +4,6 @@ authors = ["Chris Harris <[email protected]> and contributors"]
 version = "0.1.0"
 
 [deps]
-Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 IsURL = "ceb4388c-583f-448d-bb30-00b11e8c5682"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"

flake.nix

@@ -14,7 +14,8 @@
             awscli2
           ];
           shellHook = ''
-            export AWS_PROFILE=personal
+            [ -z "''${AWS_PROFILE}" ] && export AWS_PROFILE=personal
+            command -v fish &> /dev/null && fish
           '';
         };
       });

src/BuildDockerfile.jl

@@ -1,15 +1,12 @@
-const runtime_path = "/var/runtime"
-const julia_depot_path = "/var/runtime/julia_depot"
-const temp_path = "/tmp"
-const jot_github_url = "https://github.com/harris-chris/Jot.jl#main"
-
 function dockerfile_add_julia_image(julia_base_version::String)::String
   """
   FROM julia:$julia_base_version
   """
 end
 
-function dockerfile_add_additional_registries(additional_registries::Vector{String})::String
+function dockerfile_add_additional_registries(
+    additional_registries::Vector{String}
+  )::String
   using_pkg_script = "using Pkg; "
   add_registries_script = begin
     str = ""
@@ -31,7 +28,11 @@ function dockerfile_add_utilities()::String
   """
 end
 
-function dockerfile_add_runtime_directories()::String
+function dockerfile_add_runtime_directories(
+    julia_depot_path::String,
+    temp_path::String,
+    runtime_path::String
+  )::String
   """
   RUN mkdir -p $julia_depot_path
   ENV JULIA_DEPOT_PATH=$julia_depot_path
@@ -48,7 +49,10 @@ function dockerfile_copy_build_dir()::String
   """
 end
 
-function dockerfile_add_responder(res::LocalPackageResponder)::String
+function dockerfile_add_responder(
+    runtime_path::String,
+    res::LocalPackageResponder,
+  )::String
   using_pkg_script = "using Pkg; "
   add_module_script = "Pkg.develop(path=\\\"$runtime_path/$(res.package_name)\\\"); "
   instantiate_script = "Pkg.instantiate(); "
@@ -81,6 +85,7 @@ function dockerfile_add_aws_rie()::String
 end
 
 function dockerfile_add_bootstrap(
+    runtime_path::String,
     package_name::String,
     function_name::String,
     ::Type{IT}
@@ -89,8 +94,9 @@ function dockerfile_add_bootstrap(
   ENV PKG_NAME=$(package_name)
   ENV FUNC_FULL_NAME=$package_name.$function_name
   ENV FUNC_PARAM_TYPE=$IT
+  ENTRYPOINT ["$runtime_path/bootstrap"]
   RUN chmod 775 . -R
-  ENTRYPOINT ["/var/runtime/bootstrap"]
+  # RUN chmod 775 $runtime_path/bootstrap -R
   """
 end
 

src/ECRRepo.jl

@@ -91,8 +91,30 @@ indicate it no longer exists.
 """
 function delete!(repo::ECRRepo)
   repo.exists || error("Repo does not exist")
+  delete_all_tags(repo)
   delete_script = get_delete_ecr_repo_script(repo.repositoryName)
-  run(`bash -c $delete_script`)
+  deleted_repo_json = readchomp(`bash -c $delete_script`)
+  deleted_repo = JSON3.read(deleted_repo_json, Dict{String, ECRRepo})["repository"]
+  if ismissing(deleted_repo) || deleted_repo != repo
+    error("Unable to delete repo $(repo.repositoryName)")
+  end
   repo.exists = false
 end
 
+function get_all_tags(repo::ECRRepo)::Vector{Pair{String, String}}
+  repo.exists || error("Repo does not exist")
+  tags_script = get_ecr_repo_tags_script(repo)
+  tags_json = readchomp(`bash -c $tags_script`)
+  as_vec = JSON3.read(tags_json, Dict{String, Vector{Dict{String, String}}})["tags"]
+  foldl(as_vec; init=Vector{Pair{String, String}}()) do acc, dt
+   [acc; collect(dt)]
+  end
+end
+
+function delete_all_tags(repo::ECRRepo)
+  all_tags = get_all_tags(repo)
+  tag_keys = map(first, all_tags)
+  delete_script = get_delete_ecr_repo_tags_script(repo, tag_keys)
+  deleted = readchomp(`bash -c $delete_script`)
+end
+

src/Jot.jl

@@ -2,7 +2,6 @@ module Jot
 
 # IMPORTS
 using Base.Filesystem
-using Dates
 using JSON3
 using OrderedCollections
 using Parameters
@@ -40,6 +39,11 @@ export LambdaComponents
 
 # CONSTANTS
 const docker_hash_limit = 12
+const runtime_path = "/var/runtime"
+const julia_depot_path = "/var/runtime/julia_depot"
+const temp_path = "/tmp"
+const jot_github_url = "https://github.com/harris-chris/Jot.jl#main"
+const writable_depot_folders = ["logs", "scratchspaces"]
 
 abstract type LambdaComponent end
 
@@ -157,8 +161,10 @@ function add_scripts_to_build_dir(
     responder::AbstractResponder,
   )
   add_to_build(content, fname) = write_to_build_dir(content, responder.build_dir, fname)
+  bootstrap_script = get_bootstrap_script(julia_depot_path, temp_path)
   bootstrap_script |> x -> add_to_build(x, "bootstrap")
-  get_init_script(package_compile, julia_cpu_target) |> x -> add_to_build(x, "init.jl")
+  init_script = get_init_script(package_compile, julia_cpu_target)
+  add_to_build(init_script, "init.jl")
   if package_compile
     get_precompile_jl(responder.package_name) |> x -> add_to_build(x, "precompile.jl")
   end
@@ -192,16 +198,19 @@ function get_dockerfile(
     *, [
     dockerfile_add_julia_image(julia_base_version),
     dockerfile_add_utilities(),
-    dockerfile_add_runtime_directories(),
+    dockerfile_add_runtime_directories(julia_depot_path, temp_path, runtime_path),
     dockerfile_add_additional_registries(responder.registry_urls),
     dockerfile_copy_build_dir(),
-    dockerfile_add_responder(responder),
+    dockerfile_add_responder(runtime_path, responder),
     dockerfile_add_labels(combined_labels),
     dockerfile_add_jot(),
     dockerfile_add_aws_rie(),
-    dockerfile_add_bootstrap(responder.package_name,
-                             String(responder.response_function),
-                             responder.response_function_param_type),
+    dockerfile_add_bootstrap(
+      runtime_path,
+      responder.package_name,
+      String(responder.response_function),
+      responder.response_function_param_type
+    ),
     dockerfile_add_precompile(package_compile),
   ]; init = "")
   dockerfile_update(generated_dockerfile)

src/LocalImage.jl

@@ -10,8 +10,8 @@
         exists::Bool = true
     end
 
-Represents a docker image on the local machine, and stores associated metadata. Should not be 
-instantiated directly. If `exists` is `true`, then the image is assumed to exit and so should be 
+Represents a docker image on the local machine, and stores associated metadata. Should not be
+instantiated directly. If `exists` is `true`, then the image is assumed to exit and so should be
 visible from utilities such as `docker image ls`.
 """
 @with_kw mutable struct LocalImage <: LambdaComponent
@@ -23,7 +23,7 @@ visible from utilities such as `docker image ls`.
   Tag::String
   exists::Bool = true
 end
-StructTypes.StructType(::Type{LocalImage}) = StructTypes.Mutable()  
+StructTypes.StructType(::Type{LocalImage}) = StructTypes.Mutable()
 Base.:(==)(a::LocalImage, b::LocalImage) = a.ID[1:docker_hash_limit] == b.ID[1:docker_hash_limit]
 
 function get_image_full_name(image::LocalImage)::String
@@ -60,7 +60,7 @@ end
         jot_generated_only::Bool = true,
       )::Vector{LocalImage}
 
-Returns a vector of `LocalImage`s, representing all locally-stored docker images. 
+Returns a vector of `LocalImage`s, representing all locally-stored docker images.
 
 `args` are passed to the call to `docker image ls`, that is used to populate this vector.
 `jot_generated_only` specifies whether to filter for jot-generated images only.
@@ -76,7 +76,7 @@ end
 
 """
     delete!(
-      image::LocalImage; 
+      image::LocalImage;
       force::Bool=false,
     )
 
@@ -87,7 +87,7 @@ function delete!(image::LocalImage; force::Bool=false)
   image.exists || error("Image does not exist")
   args = force ? ["--force"] : []
   run(`docker image rm $(image.ID) $args`)
-  image.exists = false 
+  image.exists = false
 end
 
 function is_lambda(image::LocalImage)::Bool

src/RemoteImage.jl

@@ -15,7 +15,7 @@ the RemoteImage still exists.
   ecr_repo::Union{Missing, ECRRepo} = missing
   exists::Bool = true
 end
-StructTypes.StructType(::Type{RemoteImage}) = StructTypes.Mutable()  
+StructTypes.StructType(::Type{RemoteImage}) = StructTypes.Mutable()
 Base.:(==)(a::RemoteImage, b::RemoteImage) = a.imageDigest == b.imageDigest
 
 """
@@ -45,7 +45,6 @@ function delete!(r::RemoteImage)
   delete_script = get_delete_remote_image_script(r)
   output = readchomp(`bash -c $delete_script`)
   r.exists = false
-  @debug get_remote_images(r.ecr_repo)
   if length(get_remote_images(r.ecr_repo)) == 0
     delete!(r.ecr_repo)
   end
@@ -55,7 +54,7 @@ end
 """
     get_remote_image(local_image::LocalImage)::Union{Nothing, RemoteImage}
 
-Queries AWS and returns a `RemoteImage` instance corresponding to the given `local_image`. 
+Queries AWS and returns a `RemoteImage` instance corresponding to the given `local_image`.
 
 If multiple valid images exist, this will return the first only. If none exists, returns `nothing`.
 """
@@ -71,7 +70,7 @@ end
 """
     get_remote_image(identity::AbstractString)::Union{Nothing, RemoteImage}
 
-Queries AWS and returns a `RemoteImage` instance corresponding to the given `identity` string. 
+Queries AWS and returns a `RemoteImage` instance corresponding to the given `identity` string.
 
 The identity string will attempt to match on the name of the remote image, or the image's Digest.
 

src/Scripts.jl

@@ -1,20 +1,32 @@
 using Sockets
 
-bootstrap_script = raw"""
-#!/bin/bash
-if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then
-  LOCAL="127.0.0.1:9001"
-  echo "AWS_LAMBDA_RUNTIME_API not found, starting AWS RIE on $LOCAL"
-  exec ./aws-lambda-rie /usr/local/julia/bin/julia -e "using Jot; using $PKG_NAME; start_runtime(\\\"$LOCAL\\\", $FUNC_FULL_NAME, $FUNC_PARAM_TYPE)"
-else
-  echo "AWS_LAMBDA_RUNTIME_API = $AWS_LAMBDA_RUNTIME_API, running Julia"
-  # touch /tmp/working
-  # echo "CREATED TEMPORARY FILE"
-  # exec /usr/local/julia/bin/julia -e "using Jot; println(pwd())"
-  # echo "JULIA CREATED TEMP"
-  exec /usr/local/julia/bin/julia -e "using Jot; using $PKG_NAME; start_runtime(\\\"$AWS_LAMBDA_RUNTIME_API\\\", $FUNC_FULL_NAME, $FUNC_PARAM_TYPE)"
-fi
-"""
+function get_bootstrap_script(
+    julia_depot_path::String,
+    temp_path::String,
+  )::String
+
+  bootstrap_shebang = raw"""
+  #!/bin/bash
+  """
+
+  bootstrap_env_vars = """
+  export JULIA_DEPOT_PATH=$temp_path:$julia_depot_path
+  """
+
+  bootstrap_body = raw"""
+  if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then
+    LOCAL="127.0.0.1:9001"
+    echo "AWS_LAMBDA_RUNTIME_API not found, starting AWS RIE on $LOCAL"
+    exec ./aws-lambda-rie /usr/local/julia/bin/julia -e "using Jot; using $PKG_NAME; start_runtime(\\\"$LOCAL\\\", $FUNC_FULL_NAME, $FUNC_PARAM_TYPE)"
+  else
+    echo "AWS_LAMBDA_RUNTIME_API = $AWS_LAMBDA_RUNTIME_API, running Julia"
+    exec /usr/local/julia/bin/julia -e "println(DEPOT_PATH); using Jot; using $PKG_NAME; start_runtime(\\\"$AWS_LAMBDA_RUNTIME_API\\\", $FUNC_FULL_NAME, $FUNC_PARAM_TYPE)"
+  fi
+  """
+  bootstrap_script = bootstrap_shebang * bootstrap_env_vars * bootstrap_body
+  @debug bootstrap_script
+  bootstrap_script
+end
 
 function start_lambda_server(host::String, port::Int64)
   ROUTER = HTTP.Router()
@@ -81,7 +93,7 @@ function get_init_script(
   using Jot
   Pkg.precompile()
   """
-  pc_script = """
+  package_compile_script = """
   Pkg.add(Pkg.PackageSpec(;name="PackageCompiler", version="1.7.7"))
   using PackageCompiler
   @async Jot.start_lambda_server("127.0.0.1", 9001)
@@ -92,7 +104,7 @@ function get_init_script(
                   cpu_target="$cpu_target",
                  )
   """
-  package_compile ? precomp * pc_script : precomp
+  package_compile ? precomp * package_compile_script : precomp
 end
 
 function get_precompile_jl(
@@ -103,7 +115,13 @@ function get_precompile_jl(
   using $package_name
   using Pkg
 
-  Pkg.test("$package_name")
+  try
+    Pkg.test("$package_name")
+  catch e
+    if isa(e, LoadError)
+      rethrow(e)
+    end
+  end
 
   rf(i::Int64) = i + 1
 
@@ -117,6 +135,23 @@ function get_lambda_function_tags_script(lambda_function::LambdaFunction)::Strin
   """
 end
 
+function get_ecr_repo_tags_script(ecr_repo::ECRRepo)::String
+  """
+  aws ecr list-tags-for-resource --resource-arn=$(ecr_repo.repositoryArn)
+  """
+end
+
+function get_delete_ecr_repo_tags_script(
+    ecr_repo::ECRRepo,
+    tag_keys::Vector{String}
+  )::String
+  """
+  aws ecr untag-resource \\
+       --resource-arn $(ecr_repo.repositoryArn) \\
+       --tag-keys $(join(tag_keys, " "))
+  """
+end
+
 function get_images_in_ecr_repo_script(ecr_repo::ECRRepo)::String
   """
   aws ecr list-images --repository-name=$(ecr_repo.repositoryName)

test/JotTest4/jot-test-4.jl

@@ -1,5 +1,4 @@
 using SpecialFunctions
-using PRAS # From added registry
 
 function map_log_gamma(v::Vector{N}) where {N <: Number}
   return map(loggamma, v)

test/JotTest5/jot-test-5.jl

@@ -0,0 +1,11 @@
+using Scratch
+
+function use_scratch_space(what_to_write::String)::String
+  scratch_dir = @get_scratch!("temp_scratch")
+  scratch_fpath = joinpath(scratch_dir, "scratchfile")
+  @show scratch_fpath
+  open(scratch_fpath, "w") do f
+    write(f, what_to_write)
+  end
+  "read: " * open(f->read(f, String), scratch_fpath)
+end

test/increment_vector.jl

@@ -0,0 +1 @@
+increment_vector(v::Vector{Int}) = map(x -> x + 1, v)