populate file: add insecureLinks

rsync 3.5.0 opens merge files, including per-directory .rsync-filter
files, walking the path one directory at a time, refusing symlinks that
neither root nor the user running rsync owns, and opening each directory
for reading.  It treats the files it can't open that way as missing,
without saying so.  That happens, e.g., below a home directory with mode
711, which the user may enter but not read, or behind another user's
symlink.  Such a source then got synced without its .rsync-filter rules:
excluded files got transferred, and --delete-excluded deleted nothing.

Before syncing with rsync 3.5.0 or later a source containing .rsync-filter
files, populate asks rsync which of them it reads: a dry run with
--debug=FILTER2.  If it reports an existing one as missing, the file
source type's new insecureLinks option decides: with null, the default,
populate fails with an error naming the file; with true, rsync reads such
files anyway (--insecure-links); with false, rsync syncs without them.
Both print a warning.

File scenarios sync a source through a symlink owned by nobody, with each
setting.

6 files changed, 114 insertions, 1 deletions

diff --git a/README.md b/README.md
index e70da72..c0a9c6f 100644
--- a/README.md
+++ b/README.md
@@ -332,6 +332,17 @@ Supported attributes:
   if they exist on the target. This is passed to the `--delete-excluded` option
   of rsync. Defaults to `true`.
 
+* `insecureLinks` (optional) -
+  what to do if rsync can't read `.rsync-filter` files in `path`.  rsync 3.5.0
+  and later open them more strictly than the files they transfer, and ignore
+  the ones they can't open, e.g. because a directory on the way may be
+  entered but not read, like a home directory with mode 711, or because a
+  symlink on the way belongs to neither root nor the user deploying.  By
+  default, with `null`, populating fails with an error naming the file.  With
+  `true`, rsync reads such files anyway (`--insecure-links`); with `false`,
+  rsync syncs without them.  Both print a warning.  Older versions of rsync
+  aren't affected.
+
 * `inputs` (optional) -
   inputs of the flake in this source to draw from other sources, see
   [Flake inputs from sources](#flake-inputs-from-sources).
diff --git a/lib/types/populate.nix b/lib/types/populate.nix
index 51e5430..6193a19 100644
--- a/lib/types/populate.nix
+++ b/lib/types/populate.nix
@@ -166,6 +166,10 @@
           default = true;
           type = lib.types.bool;
         };
+        insecureLinks = lib.mkOption {
+          default = null;
+          type = lib.types.nullOr lib.types.bool;
+        };
       };
     };
     flake = lib.types.submodule {
diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix
index 992ac90..6362f92 100644
--- a/pkgs/populate/default.nix
+++ b/pkgs/populate/default.nix
@@ -753,6 +753,7 @@ let
     if test -d "$source_path"; then
       source_path=$source_path/
     fi
+    ${insecureLinks config.insecureLinks}
     # Each item gets logged with what changed about it, and its size, as
     # transferred and in total.
     ${rsync}/bin/rsync \
@@ -765,6 +766,7 @@ let
           (filter: /* sh */ "--${filter.type} ${quote filter.pattern}")
           config.filters} \
         -e ${quote (ssh' target)} \
+        $insecure_links \
         --itemize-changes \
         --out-format=${quote "%i %n%L %'''b/%'''l"} \
         --info=backup,del,copy,misc,mount,name,nonreg,remove,skip,symsafe \
@@ -787,8 +789,61 @@ let
     exclude = [];
     filters = [];
     deleteExcluded = true;
+    insecureLinks = null;
   };
 
+  # rsync 3.5.0 and later open .rsync-filter files with a stricter walk than
+  # the files they transfer, and treat the ones that walk can't open as
+  # missing, without saying so.  It fails, e.g., for a directory on the way
+  # that the user may enter but not read, like a home directory with mode
+  # 711, or a symlink that neither root nor the user owns.  So rsync, when
+  # asked, tells which .rsync-filter files in the source in $source_path it
+  # reads.  If it reports existing ones as missing, setting fails with an
+  # error if it's null, syncs without them if it's false, and makes rsync
+  # open them like the files it transfers with --insecure-links if it's true,
+  # both with a warning.  Sets insecure_links to rsync's extra argument.
+  insecureLinks = setting: /* sh */ ''
+    insecure_links=
+    rsync_version=$(${rsync}/bin/rsync --version | ${coreutils}/bin/head -n 1)
+    rsync_version=$(set -- $rsync_version; echo "$3")
+    if test "$(printf '%s\n' 3.5.0 "$rsync_version" | ${coreutils}/bin/sort -V |
+               ${coreutils}/bin/head -n 1)" = 3.5.0 &&
+        test -n "$(${findutils}/bin/find "$source_path" -name .rsync-filter -print -quit)"
+    then
+      # The first existing .rsync-filter file rsync reports as missing.
+      ignored=$(
+        ${rsync}/bin/rsync --dry-run --debug=FILTER2 -F -rlptD \
+            "$source_path" "$TMPDIR"/rsync-filter-check 2>&1 |
+        while read -r line; do
+          case $line in
+            "[sender] parse_filter_file("*") [not found]")
+              file=''${line#*parse_filter_file(}
+              file=''${file%,*,*}
+              if test -e "$file"; then
+                printf '%s\n' "$file"
+                break
+              fi
+              ;;
+          esac
+        done
+      )
+      if test -n "$ignored"; then
+        ${if setting == null then /* sh */ ''
+          >&2 printf 'error: rsync %s can'"'"'t read %s, so it would sync %s without it.  This happens, e.g., if a directory on the way isn'"'"'t readable, or if a symlink on the way belongs to neither root nor the user running rsync.  Set the source'"'"'s insecureLinks to true to make rsync read it anyway, or to false to sync without it.\n' \
+              "$rsync_version" "$ignored" "$source_path"
+          exit 1
+        '' else if setting then /* sh */ ''
+          >&2 printf 'warning: rsync %s can'"'"'t read %s; it reads it with --insecure-links anyway, because insecureLinks is true.\n' \
+              "$rsync_version" "$ignored"
+          insecure_links=--insecure-links
+        '' else /* sh */ ''
+          >&2 printf 'warning: rsync %s can'"'"'t read %s, so it syncs %s without it, because insecureLinks is false.\n' \
+              "$rsync_version" "$ignored" "$source_path"
+        ''}
+      fi
+    fi
+  '';
+
   runShell = target: command:
     if isLocalTarget target
       then command
diff --git a/tests/default.nix b/tests/default.nix
index a6b9356..285ed86 100644
--- a/tests/default.nix
+++ b/tests/default.nix
@@ -21,6 +21,7 @@ let
     remoteHost = "target";
     agePath = "${pkgs.age}/bin/age";
     inherit gitHead gitFirst;
+    rsyncVersion = pkgs.rsync.version;
   };
   flakeJq = import ../pkgs/populate/flake-jq.nix;
 in
diff --git a/tests/fixtures.sh b/tests/fixtures.sh
index 833df77..b57dcc5 100755
--- a/tests/fixtures.sh
+++ b/tests/fixtures.sh
@@ -258,6 +258,16 @@ cat > /fixtures/alt2/flake.nix <<'NIX'
 { outputs = _: { greeting = "alt2"; who = "alt2"; }; }
 NIX
 
+# filtered, filtered-link: a directory with an .rsync-filter, and a symlink to
+# it that neither root nor the user deploying owns, through which rsync 3.5.0
+# and later can't read that file.
+mkdir /fixtures/filtered
+printf -- '- /garbage\n' > /fixtures/filtered/.rsync-filter
+echo keep > /fixtures/filtered/keep
+echo garbage > /fixtures/filtered/garbage
+ln -s filtered /fixtures/filtered-link
+chown -h nobody /fixtures/filtered-link
+
 # noflake: a directory without flake.nix.
 mkdir /fixtures/noflake
 echo noflake > /fixtures/noflake/README
diff --git a/tests/scenarios.nix b/tests/scenarios.nix
index 4838b74..efad151 100644
--- a/tests/scenarios.nix
+++ b/tests/scenarios.nix
@@ -26,6 +26,8 @@
   # HEAD of /fixtures/gitsrc, and its parent.
   gitHead,
   gitFirst,
+  # Version of rsync on the deploying machine.
+  rsyncVersion,
 }:
 
 let
@@ -269,7 +271,37 @@ assert !(krops.isLocalTarget (targets.remote "/"));
         targetKind = "local";
         phases = [ (builtins.head args.phases) ];
       }))
-    ];
+    ]
+    # A source whose .rsync-filter rsync 3.5.0 and later can't read, because
+    # it's reached through a symlink that neither root nor the user deploying
+    # owns.  (Running as root, the scenarios can't use the other common
+    # cause, a directory on the way that the user may enter but not read.)
+    ++ concatMap (targetKind: map (insecureLinks: let
+      affected = lib.versionAtLeast rsyncVersion "3.5.0";
+      excluded = !affected || insecureLinks == true;
+    in scenario {
+      name = "file-${targetKind}-insecureLinks" +
+        (if insecureLinks == null then "Null" else if insecureLinks then "True" else "False");
+      inherit targetKind;
+      prepare = ''
+        mkdir -p "$T"/x
+        echo old > "$T"/x/garbage
+      '';
+      phases = twice ({
+        sources.x.file = { path = "/fixtures/filtered-link"; inherit insecureLinks; };
+      } // (if affected && insecureLinks == null then {
+        expect.fail = "can't read /fixtures/filtered-link/.rsync-filter, so it would sync /fixtures/filtered-link/ without it.";
+      } else {
+        check = ''
+          expect_eq keep "$(cat "$T"/x/keep)" keep
+          ${if excluded then ''
+            test ! -e "$T"/x/garbage || fail "garbage not excluded"
+          '' else ''
+            expect_eq garbage "$(cat "$T"/x/garbage)" garbage
+          ''}
+        '';
+      }));
+    }) [ null true false ]) targetKinds;
 
   flake = matrix "flake" {
     kind = [ "git" "path" "subdir" "remote" ];