From fc14ff7abfe1cba5814161714fd18882b5888cd1 Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Wed, 15 Jan 2020 15:12:03 -0500 Subject: [PATCH 1/4] feat: support using arbitrary image tarballs Via the `services..image.tarball` option. This permits users to supply any valid image archive, potentially (but not necessarily) generated by a builder function from `pkgs.dockerTools`. --- src/nix/modules/service/image.nix | 131 +++++++++++++++++++++++++++++- 1 file changed, 128 insertions(+), 3 deletions(-) diff --git a/src/nix/modules/service/image.nix b/src/nix/modules/service/image.nix index ae5d214..a38a1c5 100644 --- a/src/nix/modules/service/image.nix +++ b/src/nix/modules/service/image.nix @@ -1,7 +1,11 @@ { pkgs, lib, config, options, ... }: let inherit (lib) + all + flip functionArgs + hasAttr + isDerivation mkOption optionalAttrs types @@ -10,7 +14,20 @@ let inherit (pkgs) dockerTools ; - inherit (types) attrsOf listOf nullOr package str unspecified bool; + inherit (types) + addCheck + attrs + attrsOf + bool + coercedTo + listOf + nullOr + oneOf + package + path + str + unspecified + ; # TODO: dummy-config is a useless layer. Nix 2.3 will let us inspect # the string context instead, so we can avoid this. @@ -18,6 +35,41 @@ let (pkgs.writeText "dummy-config.json" (builtins.toJSON config.image.rawConfig)) ]; + # Shim for `services..image.tarball` definitions that refer to + # arbitrary paths and not `dockerTools`-produced derivations. + dummyImagePackage = outPath: { + inherit outPath; + type = "derivation"; + imageName = config.image.name; + imageTag = if config.image.tag == null then "" else config.image.tag; + }; + + # Type matching the essential attributes of derivations produced by + # `dockerTools` builder functions. + imagePackageType = addCheck attrs (x: isDerivation x && (all (flip hasAttr x) [ "imageName" "imageTag" ])); + + # `coercedTo path dummyImagePackage package` looks sufficient, but it is not. + # `coercedTo` defines this `check` function: + # + # x: (coercedType.check x && finalType.check (coerceFunc x)) || finalType.check x; + # + # and this `merge` function: + # + # loc: defs: + # let + # coerceVal = val: + # if coercedType.check val then coerceFunc val + # else val; + # in finalType.merge loc (map (def: def // { value = coerceVal def.value; }) defs); + # + # Meaning that values that satisfy `finalType.check` may still be subject to + # coercion. In this case, derivations satisfy `path.check`, so will be + # coerced using the `dummyImagePackage` function. To avoid this unnecessary + # coercion, we instead force checking whether the value satisfies + # `imagePackageType.check` *first* via placing `imagePackageType` at the head + # of the list provided to `oneOf`. + imageTarballType = oneOf [ imagePackageType (coercedTo path dummyImagePackage imagePackageType) ]; + includeStorePathsWarningAndDefault = lib.warn '' You're using a version of Nixpkgs that doesn't support the includeStorePaths parameter in dockerTools.streamLayeredImage. Without this, Arion's @@ -48,6 +100,7 @@ let builtImage = buildOrStreamLayeredImage { inherit (config.image) name + tag contents includeStorePaths ; @@ -80,6 +133,16 @@ in type = nullOr package; description = '' Docker image derivation to be `docker load`-ed. + + By default, when `services..image.nixBuild` is enabled, this is + the image produced using `services..image.command`, + `services..image.contents`, and + `services..image.rawConfig`. + ''; + defaultText = lib.literalExample '' + pkgs.dockerTools.buildLayeredImage { + # ... + }; ''; internal = true; }; @@ -110,10 +173,39 @@ in default = "localhost/" + config.service.name; defaultText = lib.literalExpression or lib.literalExample ''"localhost/" + config.service.name''; description = '' - A human readable name for the docker image. + A human readable name for the Docker image. Shows up in the `docker ps` output in the `IMAGE` column, among other places. + + ::: {.important} + If you set {option}`services..image.tarball` to an arbitrary + Docker image tarball (and not, say, a derivation produced by one of the + [`dockerTools`](https://nixos.org/manual/nixpkgs/stable/#sec-pkgs-dockerTools) + builder functions), then you **must** set {option}`services..image.name` + to the name of the image in the tarball. Otherwise, Arion will not be + able to properly identify the image in the generated Docker Compose + configuration file. + ::: + ''; + }; + image.tag = mkOption { + type = nullOr str; + default = null; + description = '' + A tag to assign to the built image, or (if you specified an image archive + with `image.tarball`) the tag that arion should use when referring to + the loaded image. + + ::: {.important} + If you set {option}`services..image.tarball` to an arbitrary + Docker image tarball (and not, say, a derivation produced by one of the + [`dockerTools`](https://nixos.org/manual/nixpkgs/stable/#sec-pkgs-dockerTools) + builder functions), then you **must** set {option}`services..image.tag` + to one of the tags associated with `services..image.name` in the + image tarball. Otherwise, Arion will not be able to properly identify + the image in the generated Docker Compose configuration file. + ::: ''; }; image.contents = mkOption { @@ -162,9 +254,42 @@ in description = '' ''; }; + image.tarball = mkOption { + type = nullOr imageTarballType; + default = builtImage; + defaultText = "${builtins.storeDir}/image-built-from-service-configuration.tar.gz"; + description = '' + Docker image tarball to be `docker load`-ed. This can be a derivation + produced with one of the [`dockerTools`](https://nixos.org/manual/nixpkgs/stable/#sec-pkgs-dockerTools) + builder functions, or a Docker image tarball at some arbitrary + location. + + ::: {.note} + Using this option causes Arion to ignore most other options in the + {option}`services..image` namespace. The exceptions are + {option}`services..image.name` and {option}`services..image.tag`, + which are used when the provided {option}`services..image.tarball` + is not a derivation with the attributes `imageName` and `imageTag`. + ::: + ''; + example = lib.literalExample or lib.literalExpression '' + let + myimage = pkgs.dockerTools.buildImage { + name = "my-image"; + contents = [ pkgs.coreutils ]; + }; + in + config.services = { + myservice = { + image.tarball = myimage; + # ... + }; + } + ''; + }; }; config = lib.mkMerge [{ - build.image = builtImage; + build.image = config.image.tarball; build.imageName = config.build.image.imageName; build.imageTag = if config.build.image.imageTag != "" From d2fb84c6e09aa61f79714d7db1e0bb717da2c6d9 Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Mon, 27 Jan 2020 14:20:11 -0500 Subject: [PATCH 2/4] feat: add arbitrary Docker image tarball example and associated subtest. This tests the functionality of the newly-added `services..image.tarball` parameter. --- examples/custom-image/arion-compose.nix | 48 +++++++++++++++++++++++++ examples/custom-image/arion-pkgs.nix | 6 ++++ tests/arion-test/default.nix | 13 +++++++ 3 files changed, 67 insertions(+) create mode 100644 examples/custom-image/arion-compose.nix create mode 100644 examples/custom-image/arion-pkgs.nix diff --git a/examples/custom-image/arion-compose.nix b/examples/custom-image/arion-compose.nix new file mode 100644 index 0000000..13b5d5b --- /dev/null +++ b/examples/custom-image/arion-compose.nix @@ -0,0 +1,48 @@ +{ pkgs, ... }: + +let + webRoot = "/www"; + + webserverImage = pkgs.dockerTools.buildLayeredImage { + name = "a-webserver"; + + config = { + Entrypoint = [ + "${pkgs.darkhttpd}/bin/darkhttpd" + webRoot + ]; + + Volumes = { + "${webRoot}" = { }; + }; + }; + }; +in +{ + project.name = "custom-image"; + services = { + + webserver = { + image.tarball = webserverImage; + + # The following is essentially equivalent to + # + # { image.tarball = webserverImage; } + # + # It is included here as a demonstration of how to configure Arion to + # load a Docker image from *any* valid image tarball, not just one + # produced with a `pkgs.dockerTools` builder function. + #image.tarball = webserverImage.outPath; + #image.name = webserverImage.imageName; + #image.tag = webserverImage.imageTag; + + service.command = [ "--port" "8000" ]; + service.ports = [ + "8000:8000" # host:container + ]; + service.volumes = [ + "${pkgs.nix.doc}/share/doc/nix/manual:${webRoot}" + ]; + }; + }; +} diff --git a/examples/custom-image/arion-pkgs.nix b/examples/custom-image/arion-pkgs.nix new file mode 100644 index 0000000..69aad13 --- /dev/null +++ b/examples/custom-image/arion-pkgs.nix @@ -0,0 +1,6 @@ +# Instead of pinning Nixpkgs, we can opt to use the one in NIX_PATH +import { + # We specify the architecture explicitly. Use a Linux remote builder when + # calling arion from other platforms. + system = "x86_64-linux"; +} diff --git a/tests/arion-test/default.nix b/tests/arion-test/default.nix index a366309..961d9ce 100644 --- a/tests/arion-test/default.nix +++ b/tests/arion-test/default.nix @@ -39,6 +39,7 @@ in # Pre-build the image because we don't want to build the world # in the vm. (preEval [ ../../examples/minimal/arion-compose.nix ]).config.out.dockerComposeYaml + (preEval [ ../../examples/custom-image/arion-compose.nix ]).config.out.dockerComposeYaml (preEval [ ../../examples/full-nixos/arion-compose.nix ]).config.out.dockerComposeYaml (preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.out.dockerComposeYaml (preEval [ ../../examples/traefik/arion-compose.nix ]).config.out.dockerComposeYaml @@ -66,6 +67,18 @@ in ) machine.wait_until_fails("curl --fail localhost:8000") + # Tests + # - examples/custom-image + with subtest("custom-image"): + machine.succeed( + "rm -rf work && cp -frT ${../../examples/custom-image} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d" + ) + machine.wait_until_succeeds("curl --fail localhost:8000") + machine.succeed( + "cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down" + ) + machine.wait_until_fails("curl --fail localhost:8000") + # Tests # - running same image again doesn't require a `docker load` with subtest("docker load only once"): From 491afaa97ccf34d3e22a8f6fb810bcec99a078c6 Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Fri, 13 Oct 2023 15:43:48 -0400 Subject: [PATCH 3/4] chore: remove `build.image{,Name,Tag}` and replace its uses with `image.tarball{,.imageName,.imageTag}`. This changeset is a hard deprecation of `build.image{,Name,Tag}` and does not do any option renaming, etc., as the removed options were all marked as internal. This changeset also includes code that ensures the newly-relied-upon options are properly defined, raising user-visible assertion errors if no sane default image name or tag could be inferred. --- src/nix/modules/composition/images.nix | 44 +++++++---- src/nix/modules/service/image.nix | 105 +++++++++++-------------- 2 files changed, 76 insertions(+), 73 deletions(-) diff --git a/src/nix/modules/composition/images.nix b/src/nix/modules/composition/images.nix index c3c09a3..e4b3912 100644 --- a/src/nix/modules/composition/images.nix +++ b/src/nix/modules/composition/images.nix @@ -13,23 +13,15 @@ let addDetails = serviceName: service: builtins.addErrorContext "while evaluating the image for service ${serviceName}" - (let - inherit (service) build; - in { - imageName = build.imageName or service.image.name; - imageTag = - if build.image.imageTag != "" - then build.image.imageTag - else lib.head (lib.strings.splitString "-" (baseNameOf build.image.outPath)); - } // (if build.image.isExe or false - then { - imageExe = build.image.outPath; + ( + let + imageAttrName = "image${lib.optionalString (service.image.tarball.isExe or false) "Exe"}"; + in + { + inherit (service.image.tarball) imageName imageTag; + ${imageAttrName} = service.image.tarball.outPath; } - else { - image = build.image.outPath; - } - ) - ); + ); in { options = { @@ -40,6 +32,26 @@ in }; }; config = { + assertions = + let + assertionsForRepoTagComponent = component: attrName: + lib.mapAttrsToList + (name: value: { + assertion = lib.types.nonEmptyStr.check value.${attrName}; + message = lib.replaceStrings [ "\n" ] [ " " ] '' + Unable to infer the ${component} of the image associated with + config.services.${name}. Please set + config.services.${name}.image.${attrName} to a non-empty + string. + ''; + }) + serviceImages; + + nameAssertions = assertionsForRepoTagComponent "name" "imageName"; + tagAssertions = assertionsForRepoTagComponent "tag" "imageTag"; + in + nameAssertions ++ tagAssertions; + build.imagesToLoad = lib.attrValues serviceImages; docker-compose.extended.images = config.build.imagesToLoad; }; diff --git a/src/nix/modules/service/image.nix b/src/nix/modules/service/image.nix index a38a1c5..c8c7b3c 100644 --- a/src/nix/modules/service/image.nix +++ b/src/nix/modules/service/image.nix @@ -21,6 +21,7 @@ let bool coercedTo listOf + nonEmptyStr nullOr oneOf package @@ -35,14 +36,42 @@ let (pkgs.writeText "dummy-config.json" (builtins.toJSON config.image.rawConfig)) ]; + # Neither image names nor tags can can be empty strings; setting either to an + # empty string will cause `docker load` to croak with the error message + # "invalid reference format". + fallbackImageRepoTagComponent = component: fallback: + if nonEmptyStr.check component + then component + else fallback; + fallbackImageName = fallbackImageRepoTagComponent config.image.name; + fallbackImageTag = fallbackImageRepoTagComponent config.image.tag; + # Shim for `services..image.tarball` definitions that refer to # arbitrary paths and not `dockerTools`-produced derivations. - dummyImagePackage = outPath: { - inherit outPath; - type = "derivation"; - imageName = config.image.name; - imageTag = if config.image.tag == null then "" else config.image.tag; - }; + dummyImagePackage = outPath: + let + tarballSuffix = ".tar.gz"; + repoTagSeparator = "-"; + baseName = baseNameOf outPath; + baseNameNoExtension = lib.strings.removeSuffix tarballSuffix baseName; + baseNameComponents = lib.strings.splitString repoTagSeparator baseNameNoExtension; + fallbacks = + if ((lib.isStorePath outPath) && (lib.hasSuffix tarballSuffix baseName) && (lib.hasInfix repoTagSeparator baseName)) + then { + imageName = lib.concatStringsSep repoTagSeparator (lib.tail baseNameComponents); + imageTag = lib.head baseNameComponents; + } + else { + imageName = null; + imageTag = null; + }; + in + { + inherit outPath; + type = "derivation"; + imageName = fallbackImageName fallbacks.imageName; + imageTag = fallbackImageTag fallbacks.imageTag; + }; # Type matching the essential attributes of derivations produced by # `dockerTools` builder functions. @@ -99,11 +128,11 @@ let builtImage = buildOrStreamLayeredImage { inherit (config.image) - name tag contents includeStorePaths ; + name = fallbackImageName ("localhost/" + config.service.name); config = config.image.rawConfig; maxLayers = 100; @@ -129,33 +158,6 @@ let in { options = { - build.image = mkOption { - type = nullOr package; - description = '' - Docker image derivation to be `docker load`-ed. - - By default, when `services..image.nixBuild` is enabled, this is - the image produced using `services..image.command`, - `services..image.contents`, and - `services..image.rawConfig`. - ''; - defaultText = lib.literalExample '' - pkgs.dockerTools.buildLayeredImage { - # ... - }; - ''; - internal = true; - }; - build.imageName = mkOption { - type = str; - description = "Derived from `build.image`"; - internal = true; - }; - build.imageTag = mkOption { - type = str; - description = "Derived from `build.image`"; - internal = true; - }; image.nixBuild = mkOption { type = bool; description = '' @@ -169,9 +171,8 @@ in ''; }; image.name = mkOption { - type = str; - default = "localhost/" + config.service.name; - defaultText = lib.literalExpression or lib.literalExample ''"localhost/" + config.service.name''; + type = nullOr str; + default = null; description = '' A human readable name for the Docker image. @@ -264,6 +265,11 @@ in builder functions, or a Docker image tarball at some arbitrary location. + By default, when `services..image.nixBuild` is enabled, this is + the image produced using `services..image.command`, + `services..image.contents`, and + `services..image.rawConfig`. + ::: {.note} Using this option causes Arion to ignore most other options in the {option}`services..image` namespace. The exceptions are @@ -272,35 +278,20 @@ in is not a derivation with the attributes `imageName` and `imageTag`. ::: ''; - example = lib.literalExample or lib.literalExpression '' - let - myimage = pkgs.dockerTools.buildImage { - name = "my-image"; - contents = [ pkgs.coreutils ]; - }; - in - config.services = { - myservice = { - image.tarball = myimage; - # ... - }; - } + example = lib.literalExpression '' + pkgs.dockerTools.buildLayeredImage { + # ... + }; ''; }; }; config = lib.mkMerge [{ - build.image = config.image.tarball; - build.imageName = config.build.image.imageName; - build.imageTag = - if config.build.image.imageTag != "" - then config.build.image.imageTag - else lib.head (lib.strings.splitString "-" (baseNameOf config.build.image.outPath)); image.rawConfig.Cmd = config.image.command; image.nixBuild = lib.mkDefault (priorityIsDefault options.service.image); } ( lib.mkIf (config.service.build.context == null) { - service.image = lib.mkDefault "${config.build.imageName}:${config.build.imageTag}"; + service.image = lib.mkDefault "${config.image.tarball.imageName}:${config.image.tarball.imageTag}"; }) ]; } From a891633e2ae27caad675c2bf06d716203123e3ac Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Sat, 14 Oct 2023 11:12:32 -0400 Subject: [PATCH 4/4] feat: add module options tests --- tests/flake-module.nix | 4 ++ tests/module-options-arion-test/default.nix | 76 +++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 tests/module-options-arion-test/default.nix diff --git a/tests/flake-module.nix b/tests/flake-module.nix index 09f1c01..66949f1 100644 --- a/tests/flake-module.nix +++ b/tests/flake-module.nix @@ -31,6 +31,10 @@ modules = [ ../examples/minimal/arion-compose.nix ]; }; + testModuleOptions = import ./module-options-arion-test { + inherit lib; + pkgs = final; + }; }; }; } diff --git a/tests/module-options-arion-test/default.nix b/tests/module-options-arion-test/default.nix new file mode 100644 index 0000000..cb99e07 --- /dev/null +++ b/tests/module-options-arion-test/default.nix @@ -0,0 +1,76 @@ +{ pkgs, lib ? pkgs.lib, ... }: +let + inherit (lib) any escapeShellArg replaceStrings runTests toList; + + evalComposition = modules: import ../../src/nix/eval-composition.nix { + inherit pkgs; + modules = toList modules; + }; + + testComposition = { expected, fn, config }: { + inherit expected; + expr = fn (evalComposition config); + }; + + search = pattern: str: (builtins.match ".*${pattern}.*" str) != null; + + assertionsMatch = patterns: assertions: + let + failed = builtins.filter (assertion: !assertion.assertion) assertions; + matchAnyPattern = assertion: any (pattern: search pattern assertion.message) (toList patterns); + in + any matchAnyPattern failed; + + checkAssertions = expected: patterns: config: testComposition { + inherit expected config; + fn = composition: assertionsMatch patterns composition.config.assertions; + }; + + checkAssertionsMatch = checkAssertions true; + checkAssertionsDoNotMatch = checkAssertions false; + + mkRepoTagAssertionPattern = component: attrName: name: replaceStrings [ "\n" ] [ " " ] '' + Unable to infer the ${component} of the image associated with + config\.services\.${name}\. Please set + config\.services\.${name}\.image\.${attrName} to a non-empty string\. + ''; + + imageNameAssertionPattern = mkRepoTagAssertionPattern "name" "imageName"; + imageTagAssertionPattern = mkRepoTagAssertionPattern "tag" "imageTag"; + + tests = runTests { + testNoImageName = checkAssertionsMatch (imageNameAssertionPattern "no-name") { + services.no-name.image = { + tarball = "/no/name.tar.gz"; + tag = "test"; + }; + }; + + testNoImageTag = checkAssertionsMatch (imageTagAssertionPattern "no-tag") { + services.no-tag.image = { + tarball = "/no/tag.tar.gz"; + name = "test"; + }; + }; + + testImageNameInference = testComposition { + config = { + services.nix-store-path.image.tarball = builtins.storeDir + "/foo-bar-baz.tar.gz"; + }; + expected = "bar-baz"; + fn = composition: composition.config.services.nix-store-path.image.tarball.imageName; + }; + + testImageTagInference = testComposition { + config = { + services.nix-store-path.image.tarball = builtins.storeDir + "/foo-bar-baz.tar.gz"; + }; + expected = "foo"; + fn = composition: composition.config.services.nix-store-path.image.tarball.imageTag; + }; + }; +in +# Abort if `tests`(list containing failed tests) is not empty +pkgs.runCommandNoCC "module-options-arion-test" { } '' + ${pkgs.jq}/bin/jq 'if . == [] then . else halt_error(1) end' > "$out" <<<${escapeShellArg (builtins.toJSON tests)} +''