diff options
| -rw-r--r-- | README.md | 6 | ||||
| -rw-r--r-- | lib/default.nix | 21 | ||||
| -rw-r--r-- | pkgs/populate/default.nix | 119 | ||||
| -rw-r--r-- | tests/scenarios.nix | 65 |
4 files changed, 154 insertions, 57 deletions
@@ -297,9 +297,9 @@ whatever is at the target path of a source, and with `prune` it removes everything else. It doesn't change or remove anything outside: * Target paths must be absolute, must not be the root directory, and must not - contain `..`. Source names must be file names other than `.`, `..`, and - `.populate`, without slashes and control characters. Anything else fails - when evaluating. + contain `..`. Source names must consist of letters, digits, and the + characters `._-`, not start with `-`, and be neither `.`, `..`, nor + `.populate`. Anything else fails when evaluating. * A symlink at the target path of a source gets replaced by what the source delivers, not followed, and populate says so. To keep a source elsewhere, diff --git a/lib/default.nix b/lib/default.nix index 0a99f97..0ffdff2 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -114,6 +114,27 @@ let { in { quote = x: if x == "" then "''" else lib.stringAsChars quoteChar x; + # Shell code defining the function url_path, which prints the path $1 + # as it goes into a URL: with every byte percent-encoded that isn't a + # letter, a digit, or one of -._~/. Nix decodes that, and otherwise + # takes, e.g., ? and # for delimiters, and refuses blanks. + urlPath = /* sh */ '' + url_path() { + printf %s "$1" | od -An -v -tx1 | tr -s ' ' '\n' | + while read -r url_byte; do + case $url_byte in + "") ;; + 2[d-f]|3[0-9]|4[1-9a-f]|5[0-9a]|5f|6[1-9a-f]|7[0-9a]|7e) + printf "\\$(printf %o "0x$url_byte")" + ;; + *) + printf '%%%s' "$url_byte" + ;; + esac + done + } + ''; + # Shell code that enables flakes for the Nix commands following it, and # fails unless Nix supports them, i.e. is version 2.4 or later. `what` # names what needs flakes, for the error message. diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index 97c2c4b..ce17305 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -6,11 +6,13 @@ with shell; let # Populate replaces and removes what's at the target paths of sources, so # it refuses names that lead out of the target directory, to the directory - # itself, or to its own state. + # itself, or to its own state. The remaining names are portable file + # names, which mean themselves to shells, find, and in URLs, and which the + # names of the scripts populating the sources may contain. checkSourceName = name: - if elem name [ "" "." ".." ".populate" ] || - builtins.match "[^/[:cntrl:]]+" name == null then - throw "krops: source name ${toJSON name} can't be used: it must be a file name other than ., .., and .populate, without slashes and control characters" + if elem name [ "." ".." ".populate" ] || + builtins.match "[0-9A-Za-z._][0-9A-Za-z._-]*" name == null then + throw "krops: source name ${toJSON name} can't be used: it must consist of letters, digits, and the characters ._-, not start with -, and be neither ., .., nor .populate" else true; @@ -73,6 +75,40 @@ let fi ''; + # Shell code that removes the entries of the directory dir, a shell word, + # that aren't named names. find, unlike a loop reading names, copes with + # every name, e.g. ones that end in blanks or contain newlines. Entries of + # populate's own state get removed like removeTree does. Entries of the + # target directory get removed without making them writable first, which + # would change files elsewhere that are hard links to files therein; for + # those, log is a shell word for the directory's name in the line that + # gets printed for each entry. + removeOthers = { dir, names, log ? null }: /* sh */ '' + find ${dir} -mindepth 1 -maxdepth 1 \ + ${concatMapStringsSep " " (name: "! -name ${quote name}") names} \ + -exec sh -c ' + ${optionalString (log != null) /* sh */ '' + log=$1 + shift + ''} + for path; do + ${optionalString (log != null) /* sh */ '' + >&2 printf "pruning %s/%s\n" "$log" "''${path##*/}" + ''} + ${if log != null then /* sh */ '' + rm -fR "$path" + '' else /* sh */ '' + if test -L "$path"; then + rm "$path" + else + chmod -R u+w "$path" + rm -fR "$path" + fi + ''} + done + ' sh ${optionalString (log != null) log} {} + + ''; + # Shell code that removes a symlink at path, saying so, because the link # may be there on purpose, but populate never follows it: what a source # delivers would change or remove what the link points to. @@ -191,7 +227,7 @@ let # directory may be, so paths get resolved. url = name: if generated ? ${name} - then ''"path:$(readlink -f ${quote "${flakesDir}/${name}"})"'' + then ''"path:$(url_path "$(readlink -f ${quote "${flakesDir}/${name}"})")"'' else ''"$(flake_url ${quote name})"''; check = name: source: if source == name then @@ -268,15 +304,17 @@ let printf '%s\n' "$@" # Sources outside the store can change without changing their # path. - for arg; do - case $arg in - path:/nix/store/*) ;; - path:*|git+file:*) - arg=''${arg#*:} - nix-hash --type sha256 "''${arg%%\?*}" - ;; + ${concatMapStrings (source: /* sh */ '' + drawn=$(readlink -f ${quote ( + if generated ? ${source} + then "${flakesDir}/${source}" + else "${target.path}/${source}" + )}) + case $drawn in + /nix/store/*) ;; + *) nix-hash --type sha256 "$drawn" ;; esac - done + '') (unique (drawnFrom name))} ''} } | sha256sum | cut -c-64 ) @@ -322,7 +360,7 @@ let ${overrideArgs name} # Not just the path, which Nix takes for the enclosing Git # work tree, if any. - nix flake lock "$@" "path:$work''${subdir:+?dir=$subdir}" >&2 + nix flake lock "$@" "path:$(url_path "$work")''${subdir:+?dir=$subdir}" >&2 ''} result=$(nix-store --add-fixed --recursive sha256 "$work") ${removeWork} @@ -386,7 +424,7 @@ let echo '}' } > flake.nix # Not just ., which Nix takes for the enclosing Git work tree, if any. - nix flake lock "path:$(pwd -P)" >&2 + nix flake lock "path:$(url_path "$(pwd -P)")" >&2 ) ''; process = name: @@ -394,17 +432,11 @@ let (if generated ? ${name} then flake name else if drawing ? ${name} && !(derived ? ${name}) then rewrite name else ""); - # Removes the entries of directory dir that aren't in names. - removeOthers = dir: names: /* sh */ '' + # Removes the entries of the state directory dir that sources named + # names don't need anymore, and the directory once it's empty. + removeStale = dir: names: /* sh */ '' if test -d ${quote dir}; then - ls -A ${quote dir} | while read -r name; do - case ${quote " ${toString names} "} in - *" $name "*) ;; - *) - ${removeTree ''${quote dir}/"$name"''} - ;; - esac - done + ${removeOthers { dir = quote dir; inherit names; }} if test -z "$(ls -A ${quote dir})"; then rmdir ${quote dir} fi @@ -420,6 +452,8 @@ let ${optionalString (drawing != {}) (requireFlakes "drawing flake inputs from sources")} + ${urlPath} + # Nix rewrites lock files in place, which also changes what else is # a hard link to the lock file, e.g. in a backup, or what it is a # symlink to. So the lock file of the flake in the directory $1 @@ -440,12 +474,12 @@ let fi if test -e "$path"/.git; then if test -e "$path"/.gitmodules; then - printf 'git+file:%s?submodules=1' "$path" + printf 'git+file:%s?submodules=1' "$(url_path "$path")" else - printf 'git+file:%s' "$path" + printf 'git+file:%s' "$(url_path "$path")" fi else - printf 'path:%s' "$path" + printf 'path:%s' "$(url_path "$path")" fi } @@ -483,16 +517,15 @@ let rmdir ${quote "${sentinel}/flake-inputs"} fi # Remove what sources left behind that don't need it anymore. - ${removeOthers flakesDir (attrNames generated)} - ${removeOthers derivedDir (attrNames derived)} - ${removeOthers patchesDir (attrNames patched)} + ${removeStale flakesDir (attrNames generated)} + ${removeStale derivedDir (attrNames derived)} + ${removeStale patchesDir (attrNames patched)} ''; # Removes entries of the target directory that aren't sources, garbage # collector roots of flake inputs of sources that aren't flake or # flake-input sources, and uploads left behind by interrupted runs. prune' = target: sources: let - pattern = names: concatMapStringsSep "|" quote names; flakeNames = attrNames (filterAttrs (_: source: elem source.type [ "flake" "flake-input" ]) sources); @@ -500,23 +533,14 @@ let set -efu ${useState "${target.path}/.populate" []} cd ${quote target.path} - ls -A | while read -r name; do - case $name in - ${pattern (attrNames sources ++ [ ".populate" ])}) ;; - *) - >&2 printf 'pruning %s\n' ${quote target.path}/"$name" - rm -fR ./"$name" - ;; - esac - done + ${removeOthers { + dir = "."; + names = attrNames sources ++ [ ".populate" ]; + log = quote target.path; + }} roots=.populate/flake-inputs if test -d "$roots"; then - ls -A "$roots" | while read -r name; do - case $name in - ${optionalString (flakeNames != []) "${pattern flakeNames}) ;;"} - *) rm -fR "$roots"/"$name" ;; - esac - done + ${removeOthers { dir = ''"$roots"''; names = flakeNames; }} if test -z "$(ls -A "$roots")"; then rmdir "$roots" fi @@ -1150,6 +1174,7 @@ let ${optionalString config.useChecksum /* sh */ "--checksum"} \ ${optionalString target.sudo /* sh */ "--rsync-path=\"sudo rsync\""} \ -e ${quote (ssh' target)} \ + --protect-args \ $insecure_links \ --itemize-changes \ --out-format=${quote "%i %n%L %'''b/%'''l"} \ diff --git a/tests/scenarios.nix b/tests/scenarios.nix index 3113586..849d3e3 100644 --- a/tests/scenarios.nix +++ b/tests/scenarios.nix @@ -1321,6 +1321,20 @@ assert !(krops.isLocalTarget (targets.remote "/")); } ]; })) + # Pruning entries whose names a shell's read would strip or split. + (s "prune-names" { + prepare = '' + mkdir -p "$T/ lead" "$T/trail " "$T/$(printf 'new\nline')" "$T/axb" "$T/[b]" + ''; + phases = [{ + sources = alt; + prune = true; + check = '' + # In the C locale, ls sorts by bytes, so . comes first. + expect_eq entries "$(LC_ALL=C ls -A "$T" | tr '\n' ' ')" '.populate alt ' + ''; + }]; + }) # A signal ending populate leaves neither the ssh master nor the # temporary directory. (s "interrupt" { @@ -1414,14 +1428,23 @@ assert !(krops.isLocalTarget (targets.remote "/")); } ]; }) - (modes "source-name" ({ draw, flake, ... }: { + # Source names that aren't input names, or that Nix has another use + # for, both of sources inputs get drawn from and of sources drawing them. + (modes "source-name" ({ draw, flake, ... }: let + x = from: { path = "/fixtures/filesrc"; } // draw { dep.follows = from; }; + in { phases = twice { sources = { "alt-1.0".file = "/fixtures/alt"; - x.file = { path = "/fixtures/filesrc"; } // draw { dep.follows = "alt-1.0"; }; + self.file = "/fixtures/alt"; + x.file = x "alt-1.0"; + y.file = x "self"; + "a.b_c-d".file = x "self"; }; check = '' - expect_eq msg "$(nix_eval_online ${flake "x"}#msg)" top:alt,sub + expect_eq x "$(nix_eval_online ${flake "x"}#msg)" top:alt,sub + expect_eq y "$(nix_eval_online ${flake "y"}#msg)" top:alt,sub + expect_eq a.b_c-d "$(nix_eval_online ${flake "a.b_c-d"}#msg)" top:alt,sub ''; }; })) @@ -1661,13 +1684,16 @@ assert !(krops.isLocalTarget (targets.remote "/")); ]) # Symlinks to outside among what populate removes: leftovers of # interrupted runs, state of sources that are gone, and, when pruning, - # entries of the target directory. They get removed as links. + # entries of the target directory. They get removed as links. So do + # hard links to a read-only file outside, whose mode stays. (s "leftovers" { isolate = true; prepare = '' - mkdir -p "$T" + mkdir -p "$T"/hard ln -s ${outside}/dir "$T"/other ln -s ${outside}/repo "$T"/repo + ln ${outside}/dir/sub/file "$T"/hard/link + ln ${outside}/dir/sub/file "$T"/hardlink ''; phases = [ { @@ -1697,13 +1723,38 @@ assert !(krops.isLocalTarget (targets.remote "/")); sources = stateful; prune = true; check = '' - for name in other repo .populate/flake-inputs/gone .populate/flake-uploads; do + for name in other repo hard hardlink .populate/flake-inputs/gone .populate/flake-uploads; do test ! -L "$T"/$name && test ! -e "$T"/$name || fail "$name not pruned" done ''; } ]; }) + # A target path with characters that mean something to shells and in + # URLs. + (s "odd-target" (let + targetPath = "/var/krops/odd-target-${targetKind}/a b*c?d#e%f"; + in { + isolate = true; + phases = twice { + sources = stateful // { + g.git = { url = gitUrl; ref = gitHead; }; + p.pipe = "/fixtures/pipe"; + }; + inherit targetPath; + prune = true; + check = '' + D=${lib.escapeShellArg targetPath} + test -e "$D"/alt/flake.nix || fail "alt missing" + expect_eq HEAD "$(git -C "$D"/g rev-parse HEAD)" ${gitHead} + expect_eq p "$(cat "$D"/p)" piped + expect_eq entries "$(ls "$T")" 'a b*c?d#e%f' + gc + expect_eq greeting "$(nix_eval "$(readlink -f "$D"/x)"#greeting)" hello+alt + expect_eq msg "$(nix_eval "$(readlink -f "$D"/y)"#msg)" top:dep+inner,sub + ''; + }; + })) # The target directory itself may be a symlink: what it points to is the # target directory then. (s "linked-target" { @@ -1757,7 +1808,7 @@ assert !(krops.isLocalTarget (targets.remote "/")); expect = "eval-error"; control = { ok.file = "/fixtures/alt"; }; }) - [ "" "." ".." ".populate" "a/b" "../x" "/etc" "a\nb" ]; + [ "" "." ".." ".populate" "a/b" "../x" "/etc" "a\nb" "a b" "a*b" "a?b" "a+b" "a=b" "-a" ]; }) # Target paths that aren't absolute, that are the root directory, or # that resolve to another directory than they name. |