make a really fancy test harness for nix module evaluation

I've never done this before and am really proud of the code; I hope the
comments help but feel free to ask questions.

As you can see by looking at the diffs to `options.nix`, it did catch
several issues that had gotten through up to this point. I'm pretty
pleased with that. As before, `nix flake check` is all you need to do to
run it.

Change-Id: I99a550e92d7b4770e52b6aba763cff2bdc4c9287

1 files changed, 275 insertions, 0 deletions

diff --git a/checks.nix b/checks.nix
new file mode 100644
index 0000000..c20451a
--- /dev/null
+++ b/checks.nix
@@ -0,0 +1,275 @@
+{ pkgs, self, system, ... }:
+
+let
+    # The copy of `lib` from `pkgs` is not quite the same as the `lib` that
+    # modules get. We're pretending this file is analogous to a package
+    # definition, so this is the only one we have available. It's declared
+    # explicitly here because doing this in `flake.nix` would have been harder
+    # to explain.
+    lib = pkgs.lib;
+
+    # `mkNixEvalCheck` is a helper function that defines a check which
+    # evaluates some nix code, and diffs the evaluation result against an
+    # expected result.
+    #
+    # While you may occasionally call this directly, it is more likely that
+    # you want `mkNixModuleCheck`. This one is presented first for ease of
+    # understanding how they work.
+    #
+    # To do this, it sets up a nested instance of `nix` which runs within the
+    # sandbox and can't touch the outer system. This is slightly involved, so
+    # this helper is its own thing even though in practice we will also need
+    # module handling logic for everything we're interested in testing.
+    mkNixEvalCheck = name: input: expected:
+        # Since this is a convenience wrapper, the implementation is trivial.
+        mkNixEvalCheckWith { inherit name input expected; };
+
+    # `mkNixEvalCheckWith` is the underlying implmentation of
+    # `mkNixEvalCheck`. It has extra options, which are needed by
+    # `mkNixModuleCheck` and likely not by anything else.
+    mkNixEvalCheckWith =
+          {
+            # These parameters have the same meanings as in
+            # `mkNixEvalCheck`.
+            name, input, expected,
+
+            # An attribute set of additional source files to link in so the
+            # builder can use them, keyed by their filenames.
+            extraSources ? { }
+          }:
+
+          let
+              # Optionally, `input` can take a `sources` parameter to find
+              # other files it relies on. We apply that parameter here.
+              #
+              # Making this work correctly is a lot of code, especially
+              # compared to just using `builtins.readFile`. It's worth it,
+              # because it makes any stack traces produced during testing
+              # significantly more readable, by adding filenames to them.
+              appliedInput = if builtins.isFunction input
+                               then input sourceFiles
+                               else input;
+
+              # An attribute set of names and file contents in the store for
+              # data that should be available to the build script via `src`
+              # and to `appliedInput` via `sources`.
+              #
+              # This family of three variables exists so that `appliedInput`
+              # doesn't need to have a cyclic dependency.
+              sourceContents = extraSources // {
+                                 "input.nix" = appliedInput;
+                                 "expected.json" = "${expected}\n";
+                               };
+
+              # An attribute set of names and derivations that should be
+              # available to the build script via `mkDerivation`'s `src`
+              # parameter.
+              #
+              # This family of three variables exists so that `appliedInput`
+              # doesn't need to have a cyclic dependency.
+              sourceDerivations =
+                  lib.mapAttrs (name: value: pkgs.writeTextDir name value)
+                               sourceContents;
+
+              # An attribute set of names and complete file paths in the store
+              # that should be available to `appliedInput` via its `sources`
+              # parameter.
+              #
+              # This family of three variables exists so that `appliedInput`
+              # doesn't need to have a cyclic dependency.
+              sourceFiles =
+                  lib.mapAttrs (name: value: "${value}/${name}")
+                               sourceDerivations;
+
+          in pkgs.stdenv.mkDerivation {
+            name = "secrets-nix-test-${name}";
+
+            # The easiest way to get the test's input into a format the
+            # build script can work with is to use it as the `src`.
+            #
+            # We construct a source directory with two files `input` and
+            # `expected`. We also respect any additional files from
+            # `extraSources`.
+            src = pkgs.symlinkJoin {
+              name = "secrets-nix-test-${name}-src";
+              paths = lib.foldlAttrs
+                          (result: name: value: result ++ [ value ])
+                          [ ]
+                          sourceDerivations;
+            };
+
+            nativeBuildInputs = with pkgs; [ diffutils nix ];
+
+            # Recall that a nix flake check is a derivation; the check
+            # succeeds if and only if the derivation builds successfully.
+            # Sadly, `checkPhase` is skipped, so we do all the work in
+            # `buildPhase`.
+            buildPhase = ''
+              # Because we potentially build some derivations, we need a
+              # proper temporary directory for the store; we can't just use
+              # the dummy store.
+              mkdir nix-store
+              ${pkgs.nix}/bin/nix \
+                  --extra-experimental-features nix-command \
+                  --store /build/nix-store \
+                  --show-trace \
+                  eval --json --file $src/input.nix > $out
+
+              # The exit code of diff is what we want for this. Yay!
+              if ! ${pkgs.diffutils}/bin/diff $src/expected.json $out; then
+                # It's good to keep the user-programmer in the loop...
+                echo
+                echo "This is a nix evaluation test case. The expected eval"
+                echo "output differed from the actual output. In an ideal"
+                echo "world, the above diff would help you understand why."
+                echo
+                false
+              fi
+            '';
+          };
+
+
+    # `mkNixModuleCheck` builds on the behavior of `mkNixEvalCheck`,
+    # attempting to call `evalModules` on a user-provided module, combining it
+    # with the smalltech options module and with an appropriate version of
+    # nixpkgs itself.
+    #
+    # This helper is intended to be useful for writing test cases that
+    # simulate configuring a complete system. It is the helper we are likely
+    # to use most frequently.
+    #
+    # It defers to `mkNixModuleCheckWith` for all the real work, so
+    # as to be able to provide some sensible defaults while still being
+    # configurable.
+    mkNixModuleCheck =
+        # This name will be appended to the derivation name. Set it to
+        # something distinctive; it appears in error messages.
+        name:
+
+        # This should be a string (usually written as a multiline string
+        # literal) giving the exact text of the input module.
+        #
+        # Unfortunately it needs to be a string, not an arbitrary nix
+        # expression, because it needs to be serialized so the inner nix can
+        # get a copy of it.
+        input:
+
+        # This should be a nix expression evaluating to an attribute set,
+        # corresponding to the expected output of the test case.
+        #
+        # It will be serialized to JSON before comparing it against the actual
+        # output.
+        expected:
+
+        # Since this is a convenience wrapper, the implementation is trivial.
+        mkNixModuleCheckWith { inherit name input expected; };
+
+
+      # `mkNixModuleCheckWith` is the underlying implementation of
+      # `mkNixModuleCheck`. The only situation in which you'd use it directly
+      # is if you need to set a custom `keyPath`.
+      #
+      # Please notice the somewhat subtle implementation; this is generating
+      # a text file containing nix code, which is interpreted by the nested
+      # nix. That's why we have to go to so much trouble to pass things around
+      # as strings.
+      #
+      # Fortunately for passing `nixpkgs` itself, we can reference
+      # the path to where it's already present in the filesystem as an input
+      # to this flake in the outer nix; otherwise the test's input file would
+      # be many megabytes in size.
+      #
+      # Less fortunately, for `options.nix` and the test input, we need to
+      # include the contents textually. This leads to some awkwardness and
+      # limitations, but it should be reliable.
+      mkNixModuleCheckWith =
+          {
+            # These parameters have the same meanings as in
+            # `mkNixModuleCheck`.
+            name, input, expected,
+
+            # Since the return value of a module potentially contains
+            # functions and other things we can't serialize, we always extract
+            # a sub-value from within it, and compare against that.
+            #
+            # The default is likely to be all we need for most cases.
+            keyPath ? "config.secrets.export"
+          }: mkNixEvalCheckWith {
+            inherit name;
+
+            extraSources = {
+              "options.nix" = builtins.readFile ./options.nix;
+              "test-input-module.nix" = input;
+            };
+
+            # We rely on the `sources` parameter; for details of how it works,
+            # see the binding of `appliedInput` in the definition of
+            # `mkNixEvalCheckWith`, above.
+            #
+            # It's important that the copy of `pkgs` available to the modules
+            # in the inner nix be the same one this file is using for its own
+            # needs, here in the outer nix. Otherwise, things like
+            # `${pkgs.bash}` in expected-output values won't work correctly,
+            # and since scripts are a critical part of the output, that's
+            # mandatory.
+            #
+            # The way it's passed through here should work, but if it breaks,
+            # well, hopefully this comment saved you some time.
+            input = sources: ''
+              let pkgs = import ${self.inputs.nixpkgs} { };
+              in (pkgs.lib.evalModules {
+                    modules = [
+                      {
+                        imports = [
+                          ${sources."options.nix"}
+                          ${sources."test-input-module.nix"}
+                        ];
+                      }
+                    ];
+                    specialArgs = { inherit pkgs; };
+                 }).${keyPath}
+            '';
+
+           expected = builtins.toJSON expected;
+         };
+
+in {
+  # Verify that the nix evaluator test harness works.
+  trivial = mkNixEvalCheck "trivial" "1 + 2" "3";
+
+  # Verify that when no secrets are configured, we correctly export a set
+  # containing no secrets.
+  empty = mkNixModuleCheck "empty" ''
+    {
+      config = { };
+    }
+  '' { };
+
+  # Verify that when a single specific secret is configured, we correctly
+  # export a set containing a description of it, and nothing else.
+  single = mkNixModuleCheck "single" ''
+    {
+      config = {
+        secrets.secrets.example = {
+          filename = "example.key";
+          script = '''
+            touch example.key
+          ''';
+        };
+      };
+    }
+  '' {
+    example = {
+      path = "/etc/nixos/secrets/example.key";
+      script = ''
+        #!${pkgs.bash}/bin/bash
+        touch example.key
+      '';
+    };
+  };
+
+  ### ADD MORE NIX MODULE TEST CASES ABOVE THIS LINE! :)
+
+  # Verify that the Rust command-line tool builds correctly.
+  rust = self.packages.${system}.default;
+}