populate file: check the .rsync-filter files that really apply

rsync reads .rsync-filter files in the directories above a source as
well, and their rules apply to the transfer.  The check for files that
rsync 3.5.0 and later can't read only starts if the source itself
contains one, so a subdirectory of a repository with a filter file at
its top gets synced without it, silently.  The check also starts for
filter files above the source.

The check's dry run is the transfer it stands for, as far as filtering
goes:

- It gets the source's exclude and filters, so it doesn't enter
  directories the transfer skips.
- It doesn't run for transfers that apply no .rsync-filter files, i.e.
  uploads of flakes and patches.
- A filter file the user may not read anyway doesn't count: rsync fails
  on its own when it gets to transfer that file, and --insecure-links
  changes nothing about it.
- rsync prints bytes it takes for unprintable as \#ooo, which depends on
  the locale.  File names get decoded before checking that they exist, so
  files below, e.g., directories with umlauts in their names don't pass
  for missing in the C locale.
- If rsync doesn't tell which filter files it reads at all, populate
  warns that it can't tell either.

The README mentions that --insecure-links also affects the way to the
target path on local targets, and that only rsync 3.5.0 ignores filter
files below directories that may be entered but not read.

Phases of scenarios can set environment variables for populate.  New
file scenarios sync a subdirectory whose filter file is above it, and a
source with a filter file below a directory with an umlaut in its name,
in the C locale.

1 files changed, 90 insertions, 32 deletions

diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix
index 2811e27..c8229fb 100644
--- a/pkgs/populate/default.nix
+++ b/pkgs/populate/default.nix
@@ -1140,24 +1140,21 @@ let
     if test -d "$source_path"; then
       source_path=$source_path/
     fi
-    ${insecureLinks config.insecureLinks}
+    # Without .rsync-filter files applying, rsync reads none.
+    ${if config.filterFiles or true
+      then insecureLinks config
+      else "insecure_links="}
     # Each item gets logged with what changed about it, and its size, as
     # transferred and in total.
     ${rsync}/bin/rsync \
         ${optionalString config.useChecksum /* sh */ "--checksum"} \
         ${optionalString target.sudo /* sh */ "--rsync-path=\"sudo rsync\""} \
-        ${concatMapStringsSep " "
-          (pattern: /* sh */ "--exclude ${quote pattern}")
-          config.exclude} \
-        ${concatMapStringsSep " "
-          (filter: /* sh */ "--${filter.type} ${quote filter.pattern}")
-          config.filters} \
         -e ${quote (ssh' target)} \
         $insecure_links \
         --itemize-changes \
         --out-format=${quote "%i %n%L %'''b/%'''l"} \
         --info=backup,del,copy,misc,mount,name,nonreg,remove,skip,symsafe \
-        ${optionalString (config.filterFiles or true) "-F"} \
+        ${rsyncFilterArgs config} \
         -rlptD \
         ${optionalString config.deleteExcluded /* sh */ "--delete-excluded"} \
         "$source_path" \
@@ -1179,41 +1176,102 @@ let
     insecureLinks = null;
   };
 
+  # rsync's arguments for what a source excludes and filters, followed by
+  # -F, which applies .rsync-filter files, unless the config turns that off.
+  rsyncFilterArgs = config: /* sh */ ''
+    ${concatMapStringsSep " "
+      (pattern: /* sh */ "--exclude ${quote pattern}")
+      config.exclude} \
+    ${concatMapStringsSep " "
+      (filter: /* sh */ "--${filter.type} ${quote filter.pattern}")
+      config.filters} \
+    ${optionalString (config.filterFiles or true) "-F"}'';
+
   # rsync 3.5.0 and later open .rsync-filter files with a stricter walk than
   # the files they transfer, and treat the ones that walk can't open as
   # missing, without saying so.  It fails, e.g., for a directory on the way
   # that the user may enter but not read, like a home directory with mode
   # 711, or a symlink that neither root nor the user owns.  So rsync, when
-  # asked, tells which .rsync-filter files in the source in $source_path it
-  # reads.  If it reports existing ones as missing, setting fails with an
-  # error if it's null, syncs without them if it's false, and makes rsync
-  # open them like the files it transfers with --insecure-links if it's true,
-  # both with a warning.  Sets insecure_links to rsync's extra argument.
-  insecureLinks = setting: /* sh */ ''
+  # asked, tells which .rsync-filter files it reads for the source in
+  # $source_path: the ones therein, and the ones in the directories above
+  # it, which apply as well.  If it reports ones as missing that exist and
+  # that the user may read, the config's insecureLinks decides: populate
+  # fails with an error if it's null, syncs without them if it's false, and
+  # makes rsync open them like the files it transfers with --insecure-links
+  # if it's true, both with a warning.  Sets insecure_links to rsync's extra
+  # argument.
+  insecureLinks = config: let
+    setting = config.insecureLinks;
+  in /* sh */ ''
     insecure_links=
     rsync_version=$(${rsync}/bin/rsync --version | ${coreutils}/bin/head -n 1)
     rsync_version=$(set -- $rsync_version; echo "$3")
+    # Whether a .rsync-filter file applies at all.
+    filtered=$(${findutils}/bin/find "$source_path" -name .rsync-filter -print -quit)
+    above=$(${coreutils}/bin/dirname "$source_path")
+    while test -z "$filtered"; do
+      if test -e "$above"/.rsync-filter; then
+        filtered=$above/.rsync-filter
+      elif test "$above" = / || test "$above" = .; then
+        break
+      fi
+      above=$(${coreutils}/bin/dirname "$above")
+    done
     if test "$(printf '%s\n' 3.5.0 "$rsync_version" | ${coreutils}/bin/sort -V |
                ${coreutils}/bin/head -n 1)" = 3.5.0 &&
-        test -n "$(${findutils}/bin/find "$source_path" -name .rsync-filter -print -quit)"
+        test -n "$filtered"
     then
-      # The first existing .rsync-filter file rsync reports as missing.
-      ignored=$(
-        ${rsync}/bin/rsync --dry-run --debug=FILTER2 -F -rlptD \
-            "$source_path" "$TMPDIR"/rsync-filter-check 2>&1 |
-        while read -r line; do
-          case $line in
-            "[sender] parse_filter_file("*") [not found]")
-              file=''${line#*parse_filter_file(}
-              file=''${file%,*,*}
-              if test -e "$file"; then
-                printf '%s\n' "$file"
-                break
-              fi
-              ;;
-          esac
-        done
-      )
+      # A dry run of the same transfer, as far as filtering goes.
+      ${rsync}/bin/rsync --dry-run --debug=FILTER2 \
+          ${rsyncFilterArgs config} \
+          -rlptD \
+          "$source_path" "$TMPDIR"/rsync-filter-check \
+        > "$TMPDIR"/rsync-filter-check.log 2>&1 || :
+      # The first .rsync-filter file rsync reports as missing although it
+      # exists.  One the user may not read isn't for rsync to read anyway.
+      parsed=
+      ignored=
+      while IFS= read -r line; do
+        case $line in
+          "[sender] parse_filter_file("*)
+            parsed=1
+            ;;
+        esac
+        case $line in
+          "[sender] parse_filter_file("*") [not found]")
+            file=''${line#*parse_filter_file(}
+            file=''${file%,*,*}
+            # rsync prints bytes it takes for unprintable, which depends on
+            # the locale, as \#ooo.
+            escaped=$file
+            file=
+            while :; do
+              case $escaped in
+                *'\#'[0-7][0-7][0-7]*)
+                  literal=''${escaped%%'\#'[0-7][0-7][0-7]*}
+                  escaped=''${escaped#"$literal"'\#'}
+                  byte=$(printf "\\$(printf %s "$escaped" | ${coreutils}/bin/cut -c1-3)x")
+                  file=$file$literal''${byte%x}
+                  escaped=''${escaped#???}
+                  ;;
+                *)
+                  file=$file$escaped
+                  break
+                  ;;
+              esac
+            done
+            if test -e "$file" && test -r "$file"; then
+              ignored=$file
+              break
+            fi
+            ;;
+        esac
+      done < "$TMPDIR"/rsync-filter-check.log
+      ${coreutils}/bin/rm "$TMPDIR"/rsync-filter-check.log
+      if test -z "$parsed"; then
+        >&2 printf 'warning: rsync %s doesn'"'"'t tell which .rsync-filter files it reads, so it may sync %s without some of them unnoticed.\n' \
+            "$rsync_version" "$source_path"
+      fi
       if test -n "$ignored"; then
         ${if setting == null then /* sh */ ''
           >&2 printf 'error: rsync %s can'"'"'t read %s, so it would sync %s without it.  This happens, e.g., if a directory on the way isn'"'"'t readable, or if a symlink on the way belongs to neither root nor the user running rsync.  Set the source'"'"'s insecureLinks to true to make rsync read it anyway, or to false to sync without it.\n' \