From: Scott Worley Date: Fri, 5 Jun 2020 00:25:06 +0000 (-0700) Subject: Dynamic config X-Git-Url: http://git.scottworley.com/auto-upgrade-with-pinch/commitdiff_plain/f1a53b29b8269cb5dd28a3285bc95a7df37f9a16 Dynamic config Previously, changes to the list of pinch files or the list of user environments required two updates to take effect (because these were incorporated into the update script itself). Now they take effect after one update cycle, as one would reasonably expect. Changes to the list of repos to pull take effect even if system build fails. This is intended to allow recovery from situations that would otherwise cause updates to stop. --- diff --git a/modules/auto-upgrade.nix b/modules/auto-upgrade.nix index 5ee707f..7e48c40 100644 --- a/modules/auto-upgrade.nix +++ b/modules/auto-upgrade.nix @@ -2,74 +2,60 @@ with lib; let cfg = config.system.autoUpgradeWithPinch; - pull-repo-script = path: repo: + pull-repo-script = pkgs.writeShellScript "pull-repo" '' set -eo pipefail - echo Pulling in ${escapeShellArg path} >&2 + path=$1 + config=$2 - if [[ ! -e ${escapeShellArg path} ]];then + prop() { + ${pkgs.jq}/bin/jq -r ".$1" <<< "$config" + } + + echo Pulling in "$path" >&2 + + if [[ ! -e "$path" ]];then d=$(mktemp -d) ${pkgs.git}/bin/git init "$d" - ${pkgs.git}/bin/git -C "$d" checkout -b \ - ${escapeShellArg repo.localBranch} - ${pkgs.git}/bin/git -C "$d" remote add \ - ${escapeShellArg repo.remoteName} \ - ${escapeShellArg repo.url} - ${pkgs.git}/bin/git -C "$d" branch -u \ - ${escapeShellArg repo.remoteBranch} - mkdir -p "$(dirname ${escapeShellArg path})" - mv "$d" ${escapeShellArg path} + ${pkgs.git}/bin/git -C "$d" checkout -b "$(prop localBranch)" + ${pkgs.git}/bin/git -C "$d" remote add "$(prop remoteName)" "$(prop url)" + ${pkgs.git}/bin/git -C "$d" branch -u "$(prop remoteBranch)" + mkdir -p "$(${pkgs.coreutils}/bin/dirname "$path")" + mv "$d" "$path" fi - cd ${escapeShellArg path} - - if [[ "$(${pkgs.git}/bin/git remote get-url \ - ${escapeShellArg repo.remoteName})" != \ - ${escapeShellArg repo.url} ]] - then - echo Expected git remote ${escapeShellArg repo.remoteName} \ - to point at ${escapeShellArg repo.url} \ - but it points at "$(${pkgs.git}/bin/git remote get-url \ - ${escapeShellArg repo.remoteName})" >&2 - ${ - { - abort = "exit 1"; - update = '' - echo Updating it >&2 - ${pkgs.git}/bin/git -C "$d" remote set-url \ - ${escapeShellArg repo.remoteName} \ - ${escapeShellArg repo.url} - ''; - }."${repo.onRemoteURLMismatch}" - } + cd "$path" + + if [[ "$(${pkgs.git}/bin/git remote get-url "$(prop remoteName)")" != "$(prop url)" ]]; then + echo Expected git remote "$(prop remoteName)" to point at "$(prop url)" \ + but it points at "$(${pkgs.git}/bin/git remote get-url "$(prop remoteName)")" >&2 + case "$(prop onRemoteURLMismatch)" in + abort) exit 1;; + update) echo Updating it >&2 + ${pkgs.git}/bin/git -C "$d" remote set-url "$(prop remoteName)" "$(prop url)";; + esac fi - ${pkgs.git}/bin/git fetch \ - ${escapeShellArg repo.remoteName} \ - ${escapeShellArg repo.remoteBranch} + ${pkgs.git}/bin/git fetch "$(prop remoteName)" "$(prop remoteBranch)" - if [[ "$(${pkgs.git}/bin/git rev-parse --abbrev-ref HEAD)" != \ - ${escapeShellArg repo.localBranch} ]] - then + if [[ "$(${pkgs.git}/bin/git rev-parse --abbrev-ref HEAD)" != "$(prop localBranch)" ]];then echo Could not merge because currently-checked-out \ \""$(${pkgs.git}/bin/git rev-parse --abbrev-ref HEAD)"\" is not \ - \"${escapeShellArg repo.localBranch}\" - ${ - { - abort = "exit 1"; - continue = "exit 0"; - }."${repo.onBranchMismatch}" - } + \""$(prop localBranch)"\" + case "$(prop onBranchMismatch)" in + abort) exit 1;; + continue) exit 0;; + esac fi - - ${if repo.requireSignature then '' - PATH="${pkgs.keyedgit repo.signingKeys}/bin:$PATH" \ - ${pkgs.polite-merge}/bin/polite-merge --ff-only --verify-signatures - '' else '' - ${pkgs.polite-merge}/bin/polite-merge --ff-only - ''} + if [[ "$(prop requireSignature)" == true ]]; then + ${pkgs.polite-merge}/bin/polite-merge \ + -c gpg.program='${pkgs.keyedgpg} '"$(prop 'signingKeys[]' | tr \\n ' ')"' --' \ + merge --ff-only --verify-signatures + else + ${pkgs.polite-merge}/bin/polite-merge merge --ff-only + fi ''; auto-upgrade-script = pkgs.writeShellScript "auto-upgrade" '' @@ -77,6 +63,26 @@ let pkgs.writeShellScript "auto-upgrade-with-lock-held" '' set -eo pipefail + dry_run=false + pinch_args=() + if [[ "$1" == --dry-run ]];then + dry_run=true + pinch_args=( --dry-run ) + fi + + hydrate() { + if "$dry_run";then + echo "Would run: $*" + else + "$@" + fi + } + + die() { + echo "$*" >&2 + exit 1 + } + in_tmpdir() { d=$(${pkgs.coreutils}/bin/mktemp -d) pushd "$d" @@ -85,34 +91,108 @@ let ${pkgs.coreutils}/bin/rm -r "$d" } + ${optionalString (cfg.upgradeConfigOwnershipPolicy != "any") ('' + verify_ownership() { + if [[ "$1" != /* ]];then + die "Unexpected relative path: $1" + fi + if [[ "$1" != / ]];then + verify_ownership "$(${pkgs.coreutils}/bin/dirname "$1")" + fi + if [[ ! -e "$1" ]];then + die "Could not find upgrade config: $1 does not exist" + fi + if [[ -h "$1" ]];then + verify_ownership "$( + ${pkgs.coreutils}/bin/realpath --no-symlinks \ + "$(${pkgs.coreutils}/bin/dirname "$1")/$(${pkgs.coreutils}/bin/readlink "$1")" + )" + fi + perms="$(${pkgs.findutils}/bin/find "$1" -maxdepth 0 -printf "%M")" + if [[ "$perms" == d*t ]];then + die "Will not use upgrade config in sticky directory $1" + fi + owner=$(${pkgs.findutils}/bin/find "$1" -maxdepth 0 -printf "%u") + if [[ "$owner" != root ]];then + die "Will not use upgrade config not owned by root in $1" + fi + if [[ "$perms" == l* ]];then + return 0 # Root-owned symlinks are fine + fi + if [[ "$perms" == *w? ]];then + die "Will not use world-writable upgrade config in $1" + fi + ${ + { + root = '' + if [[ "$perms" == *w???? ]];then + die "Will not use group-writable upgrade config in $1" + fi + ''; + wheel = '' + if [[ "$perms" == *w???? ]];then + group=$(${pkgs.findutils}/bin/find "$1" -maxdepth 0 -printf "%g") + if [[ "$group" != wheel ]];then + die "Will not use non-wheel-group group-writable upgrade config in $1" + fi + fi + ''; + }."${cfg.upgradeConfigOwnershipPolicy}" + } + } + '' + + concatMapStringsSep "\n" (f: "verify_ownership ${escapeShellArg f}") + cfg.upgradeConfig)} + + config=$(${pkgs.nix}/bin/nix eval --json -f ${../upgrade-config.nix} \ + --arg upgradeConfig ${ + escapeShellArg ("[" + + lib.concatMapStringsSep " " lib.strings.escapeNixString + cfg.upgradeConfig + "]") + } config) + + config_query() { + ${pkgs.jq}/bin/jq -r "$@" <<< "$config" + } + + repo_query() { + config_query --arg path "$1" ".repos[\$ARGS.named.path]$2" + } + + userenv_query() { + config_query --arg user "$1" ".userEnvironments[\$ARGS.named.user]$2" + } + # Pull updates - ${concatStringsSep "\n" (mapAttrsToList (path: repo: '' - /run/wrappers/bin/sudo -u ${escapeShellArg repo.user} \ - ${pull-repo-script path repo} - '') cfg.repos)} + while read path;do + hydrate /run/wrappers/bin/sudo -u "$(repo_query "$path" .user)" \ + ${pull-repo-script} "$path" "$(repo_query "$path" "")" + done < <( config_query '.repos | keys []' ) # Update channels - ${pkgs.pinch}/bin/pinch update ${escapeShellArgs cfg.pinchFiles} + config_query '.pinchFiles[]' | ${pkgs.findutils}/bin/xargs --no-run-if-empty --delimiter=\\n ${pkgs.pinch}/bin/pinch update "''${pinch_args[@]}" # Build - in_tmpdir ${config.system.build.nixos-rebuild}/bin/nixos-rebuild build - ${concatStringsSep "\n" (mapAttrsToList (user: env: '' - /run/wrappers/bin/sudo -u ${escapeShellArg user} \ - nix-build --no-out-link '' \ - -A ${escapeShellArg env.package} - '') cfg.userEnvironments)} + in_tmpdir hydrate ${config.system.build.nixos-rebuild}/bin/nixos-rebuild build + while read user;do + hydrate /run/wrappers/bin/sudo -u "$user" \ + ${pkgs.nix}/bin/nix-build --no-out-link '' -A "$(userenv_query "$user" .package)" + done < <( config_query '.userEnvironments | keys []' ) # Install - ${config.system.build.nixos-rebuild}/bin/nixos-rebuild switch - ${concatStringsSep "\n" (mapAttrsToList (user: env: '' - /run/wrappers/bin/sudo -u ${escapeShellArg user} \ - nix-env -f '' \ - ${optionalString (env.otherPackagesAction != "keep") "-r"} \ - -iA ${escapeShellArg env.package} - '') cfg.userEnvironments)} + hydrate ${config.system.build.nixos-rebuild}/bin/nixos-rebuild switch + while read user;do + remove_arg=-r + if [[ "$(userenv_query "$user" .otherPackagesAction)" == keep ]];then + remove_arg= + fi + hydrate /run/wrappers/bin/sudo -u "$user" \ + ${pkgs.nix}/bin/nix-env -f '' $remove_arg -iA "$(userenv_query "$user" .package)" + done < <( config_query '.userEnvironments | keys []' ) '' } ''; + in { options = { system.autoUpgradeWithPinch = { @@ -138,132 +218,28 @@ in { ''; }; - repos = mkOption { + upgradeConfig = mkOption { + type = types.listOf types.path; description = '' - Git repositories to pull before running pinch. These are maintained - as git checkouts at specified places in the filesystem with specified - ownership rather than kept read-only in the nix store so that humans - can use them both as points of intervention in the automation and to - author and push changes back up. + Configuration files that specify what git repo paths to pull, what + pinch files to update from, and what user environments to update. + These are specified in separate configuration files processed at + update time so that changes to this configuration take effect in + the same update cycle. ''; - type = types.attrsOf (types.submodule { - options = { - url = mkOption { - description = "Remote git repo."; - type = types.str; - }; - remoteName = mkOption { - description = ''Name of the git remote. Customarily "origin".''; - type = types.str; - default = "origin"; - }; - onRemoteURLMismatch = mkOption { - description = '' - What to do if the remote URL in the git repo doesn't match the - URL configured here. - ''; - type = types.enum [ "update" "abort" ]; - default = "update"; - }; - onBranchMismatch = mkOption { - description = '' - What to do if a different branch is currently checked out. - - (Changes from remoteBranch are only ever - merged into localBranch, so if a different - branch is checked out, no remote changes will be merged.) - ''; - type = types.enum [ "continue" "abort" ]; - default = "continue"; - }; - user = mkOption { - description = "User as which to run 'git fetch'"; - type = types.str; - }; - localBranch = mkOption { - description = ""; - type = types.str; - default = "master"; - }; - remoteBranch = mkOption { - type = types.str; - default = "master"; - }; - requireSignature = mkOption { - type = types.bool; - default = true; - description = '' - Only pull when the tip of the remote ref is signed by a key - specifed in signingKeys. - ''; - }; - signingKeys = mkOption { - type = types.either types.path (types.listOf types.path); - description = '' - Files containing GPG keys that are authorized to sign updates. - Updates are only merged if the commit at the tip of the remote - ref is signed with one of these keys. - ''; - }; - }; - }); - example = { - "/etc/nixos" = { - url = "https://github.com/chkno/auto-upgrade-demo-nixos"; - user = "root"; - signingKeys = [ ./admins.asc ]; - }; - "/home/alice/.config/nixpkgs" = { - url = "https://github.com/chkno/auto-upgrade-demo-user-nixpkgs"; - user = "alice"; - signingKeys = [ ./admins.asc ./alice.asc ]; - }; - }; }; - pinchFiles = mkOption { + upgradeConfigOwnershipPolicy = mkOption { + type = types.enum [ "root" "wheel" "any" ]; + default = "root"; description = '' - Pinch files to use for channel updates. Typically these are inside - repos' paths. - ''; - type = types.listOf types.path; - example = [ "/etc/nixos/channels" ]; - }; + Verify ownership of upgrade config files before using them for + system upgrades. - userEnvironments = mkOption { - description = '' - User environments to update as part of an upgrade run. + root = Config must be writable only by root. + wheel = Config must be writable only by root and wheel. + any = No checks. Not recommended. ''; - type = types.attrsOf (types.submodule { - options = { - package = mkOption { - type = types.str; - default = "userPackages"; - description = '' - The name of the single package that will be updated. You'll - want to create an 'entire user environment' package as shown in - https://nixos.wiki/wiki/FAQ#How_can_I_manage_software_with_nix-env_like_with_configuration.nix.3F - ''; - }; - otherPackagesAction = mkOption { - type = types.enum [ "remove" "keep" "abort" ]; - default = "remove"; - description = '' - What to do with packages other than package. - - THIS DEFAULTS TO "remove", WHICH IS POTENTIALLY SOMEWHAT - DESTRUCTIVE! This is the default because it is the recommended - setting -- This module recommends managing your environment - through your one entire-environment package. - This keeps your environment declarative and ensures that all - packages receive regular updates. - ''; - # It seems like "upgrade" ought to be another choice here, powered - # by "nix-env --upgrade". But when I tried this, it didn't work. - }; - }; - }); - example = { alice = { }; }; }; }; }; @@ -285,7 +261,7 @@ in { ''; nixpkgs.overlays = [ - (import ../overlays/keyedgit.nix) + (import ../overlays/keyedgpg.nix) (import ../overlays/pinch.nix) (import ../overlays/polite-merge.nix) (self: super: { diff --git a/overlays/keyedgit.nix b/overlays/keyedgit.nix deleted file mode 100644 index 6cce6fe..0000000 --- a/overlays/keyedgit.nix +++ /dev/null @@ -1,40 +0,0 @@ -# Following the instructions at https://tribut.de/blog/git-commit-signatures-trusted-keys - -self: super: { - keyedgit = keys: - let - keyfile = if builtins.isList keys then - super.runCommand "keyfile" { } '' - cat ${super.lib.escapeShellArgs keys} > $out - '' - else - keys; - homelessGPG = super.writeShellScript "homeless-gpg" '' - export GNUPGHOME=$(mktemp -d) - trap 'rm -r "$GNUPGHOME"' EXIT - ${self.gnupg}/bin/gpg "$@" - ''; - keyring = super.runCommand "keyedkeyring.gpg" { } '' - ${homelessGPG} --no-default-keyring --keyring=$out --import ${keyfile} - ''; - keyids = super.runCommand "keyids" { } '' - ${homelessGPG} --no-default-keyring --with-colons --show-keys ${keyfile} | - ${self.gawk}/bin/awk -F: '$1 == "pub" { print $5 }' > $out - ''; - keyedGPG = super.writeShellScript "keyed-gpg" '' - trusted_key_args=() - while read keyid;do - trusted_key_args+=( --trusted-key "$keyid" ) - done < ${keyids} - ${homelessGPG} --no-default-keyring --keyring=${keyring} "''${trusted_key_args[@]}" "$@" - ''; - in super.symlinkJoin { - name = "keyedgit"; - paths = [ self.git ]; - buildInputs = [ super.makeWrapper ]; - postBuild = '' - wrapProgram "$out/bin/git" \ - --add-flags '-c gpg.program=${keyedGPG}' - ''; - }; -} diff --git a/overlays/keyedgpg.nix b/overlays/keyedgpg.nix new file mode 100644 index 0000000..202abb3 --- /dev/null +++ b/overlays/keyedgpg.nix @@ -0,0 +1,54 @@ +# Following the instructions at https://tribut.de/blog/git-commit-signatures-trusted-keys +# Use with git with -c gpg.program='keyedgpg /path/to/keyfile.asc' + +self: super: +let + homelessGPG = super.writeShellScript "homeless-gpg" '' + set -eo pipefail + + export GNUPGHOME=$(${self.coreutils}/bin/mktemp -d) + trap '${self.coreutils}/bin/rm -r "$GNUPGHOME"' EXIT + ${self.gnupg}/bin/gpg --no-default-keyring "$@" + ''; +in { + keyedgpg = super.writeShellScript "keyed-gpg" '' + set -eo pipefail + + usage() { + echo "usage: keyed-gpg /path/to/keyfile1.asc ... -- gpg-command..." >&2 + exit 1 + } + + incomplete=true + keyfiles=() + while (( $# > 0 ));do + if [[ "$1" == -- ]];then + shift + incomplete=false + break + fi + if [[ ! -r "$1" ]];then + usage + fi + keyfiles+=$1 + shift + done + if "$incomplete";then + usage + fi + + keyring=$(${self.coreutils}/bin/mktemp) + cleanup() { ${self.coreutils}/bin/rm "$keyring"; } + trap cleanup EXIT + ${homelessGPG} --keyring="$keyring" --import "''${keyfiles[@]}" + + trusted_key_args=() + while read keyid;do + trusted_key_args+=( --trusted-key "$keyid" ) + done < <( + ${homelessGPG} --with-colons --show-keys "''${keyfiles[@]}" | + ${self.gawk}/bin/awk -F: '$1 == "pub" { print $5 }') + + ${homelessGPG} --keyring="$keyring" "''${trusted_key_args[@]}" "$@" + ''; +} diff --git a/overlays/polite-merge.nix b/overlays/polite-merge.nix index 49d8d3b..b1cf565 100644 --- a/overlays/polite-merge.nix +++ b/overlays/polite-merge.nix @@ -5,11 +5,11 @@ self: super: { self.callPackage ({ fetchgit, git, stdenv, }: stdenv.mkDerivation rec { pname = "polite-merge"; - version = "1.0"; + version = "2.0"; src = fetchgit { url = "https://scottworley.com/polite-merge.git"; rev = version; - sha256 = "1q3iya5ifpcnmmvxhaphlvvq674yzwkgi3cyr6i3yflqks7zf81p"; + sha256 = "0dds7pf60aqm7r4g13q1bhfbzmi478c2zy2ddj5zgq0rp2kwfaba"; }; postUnpack = "patchShebangs ."; checkInputs = [ git ]; diff --git a/upgrade-config.nix b/upgrade-config.nix new file mode 100644 index 0000000..e9962eb --- /dev/null +++ b/upgrade-config.nix @@ -0,0 +1,157 @@ +{ upgradeConfig, lib ? (import { }).lib, }: +with lib; +evalModules { + modules = upgradeConfig ++ [{ + options = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to periodically upgrade NixOS to the latest version. + Presumes that /etc/nixos is a git repo with a remote and + contains a pinch file called "channels". + ''; + }; + + dates = mkOption { + default = "04:40"; + type = types.str; + description = '' + Specification (in the format described by + systemd.time + 7) of the time at + which the update will occur. + ''; + }; + + repos = mkOption { + description = '' + Git repositories to pull before running pinch. These are maintained + as git checkouts at specified places in the filesystem with specified + ownership rather than kept read-only in the nix store so that humans + can use them both as points of intervention in the automation and to + author and push changes back up. + ''; + type = types.attrsOf (types.submodule { + options = { + url = mkOption { + description = "Remote git repo."; + type = types.str; + }; + remoteName = mkOption { + description = ''Name of the git remote. Customarily "origin".''; + type = types.str; + default = "origin"; + }; + onRemoteURLMismatch = mkOption { + description = '' + What to do if the remote URL in the git repo doesn't match the + URL configured here. + ''; + type = types.enum [ "update" "abort" ]; + default = "update"; + }; + onBranchMismatch = mkOption { + description = '' + What to do if a different branch is currently checked out. + + (Changes from remoteBranch are only ever + merged into localBranch, so if a different + branch is checked out, no remote changes will be merged.) + ''; + type = types.enum [ "continue" "abort" ]; + default = "continue"; + }; + user = mkOption { + description = "User as which to run 'git fetch'"; + type = types.str; + }; + localBranch = mkOption { + description = ""; + type = types.str; + default = "master"; + }; + remoteBranch = mkOption { + type = types.str; + default = "master"; + }; + requireSignature = mkOption { + type = types.bool; + default = true; + description = '' + Only pull when the tip of the remote ref is signed by a key + specifed in signingKeys. + ''; + }; + signingKeys = mkOption { + type = types.listOf types.path; + description = '' + Files containing GPG keys that are authorized to sign updates. + Updates are only merged if the commit at the tip of the remote + ref is signed with one of these keys. + ''; + }; + }; + }); + example = { + "/etc/nixos" = { + url = "https://github.com/chkno/auto-upgrade-demo-nixos"; + user = "root"; + signingKeys = [ ./admins.asc ]; + }; + "/home/alice/.config/nixpkgs" = { + url = "https://github.com/chkno/auto-upgrade-demo-user-nixpkgs"; + user = "alice"; + signingKeys = [ ./admins.asc ./alice.asc ]; + }; + }; + }; + + pinchFiles = mkOption { + description = '' + Pinch files to use for channel updates. Typically these are inside + repos' paths. + ''; + type = types.listOf types.path; + example = [ "/etc/nixos/channels" ]; + }; + + userEnvironments = mkOption { + description = '' + User environments to update as part of an upgrade run. + ''; + type = types.attrsOf (types.submodule { + options = { + package = mkOption { + type = types.str; + default = "userPackages"; + description = '' + The name of the single package that will be updated. You'll + want to create an 'entire user environment' package as shown in + https://nixos.wiki/wiki/FAQ#How_can_I_manage_software_with_nix-env_like_with_configuration.nix.3F + ''; + }; + otherPackagesAction = mkOption { + type = types.enum [ "remove" "keep" "abort" ]; + default = "remove"; + description = '' + What to do with packages other than package. + + THIS DEFAULTS TO "remove", WHICH IS POTENTIALLY SOMEWHAT + DESTRUCTIVE! This is the default because it is the recommended + setting -- This module recommends managing your environment + through your one entire-environment package. + This keeps your environment declarative and ensures that all + packages receive regular updates. + ''; + # It seems like "upgrade" ought to be another choice here, powered + # by "nix-env --upgrade". But when I tried this, it didn't work. + }; + }; + }); + example = { alice = { }; }; + }; + }; + }]; +}