diff options
Diffstat (limited to 'pkgs')
| -rw-r--r-- | pkgs/populate/default.nix | 122 |
1 files changed, 90 insertions, 32 deletions
diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index 2811e27..c8229fb 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -1140,24 +1140,21 @@ let if test -d "$source_path"; then source_path=$source_path/ fi - ${insecureLinks config.insecureLinks} + # Without .rsync-filter files applying, rsync reads none. + ${if config.filterFiles or true + then insecureLinks config + else "insecure_links="} # Each item gets logged with what changed about it, and its size, as # transferred and in total. ${rsync}/bin/rsync \ ${optionalString config.useChecksum /* sh */ "--checksum"} \ ${optionalString target.sudo /* sh */ "--rsync-path=\"sudo rsync\""} \ - ${concatMapStringsSep " " - (pattern: /* sh */ "--exclude ${quote pattern}") - config.exclude} \ - ${concatMapStringsSep " " - (filter: /* sh */ "--${filter.type} ${quote filter.pattern}") - config.filters} \ -e ${quote (ssh' target)} \ $insecure_links \ --itemize-changes \ --out-format=${quote "%i %n%L %'''b/%'''l"} \ --info=backup,del,copy,misc,mount,name,nonreg,remove,skip,symsafe \ - ${optionalString (config.filterFiles or true) "-F"} \ + ${rsyncFilterArgs config} \ -rlptD \ ${optionalString config.deleteExcluded /* sh */ "--delete-excluded"} \ "$source_path" \ @@ -1179,41 +1176,102 @@ let insecureLinks = null; }; + # rsync's arguments for what a source excludes and filters, followed by + # -F, which applies .rsync-filter files, unless the config turns that off. + rsyncFilterArgs = config: /* sh */ '' + ${concatMapStringsSep " " + (pattern: /* sh */ "--exclude ${quote pattern}") + config.exclude} \ + ${concatMapStringsSep " " + (filter: /* sh */ "--${filter.type} ${quote filter.pattern}") + config.filters} \ + ${optionalString (config.filterFiles or true) "-F"}''; + # rsync 3.5.0 and later open .rsync-filter files with a stricter walk than # the files they transfer, and treat the ones that walk can't open as # missing, without saying so. It fails, e.g., for a directory on the way # that the user may enter but not read, like a home directory with mode # 711, or a symlink that neither root nor the user owns. So rsync, when - # asked, tells which .rsync-filter files in the source in $source_path it - # reads. If it reports existing ones as missing, setting fails with an - # error if it's null, syncs without them if it's false, and makes rsync - # open them like the files it transfers with --insecure-links if it's true, - # both with a warning. Sets insecure_links to rsync's extra argument. - insecureLinks = setting: /* sh */ '' + # asked, tells which .rsync-filter files it reads for the source in + # $source_path: the ones therein, and the ones in the directories above + # it, which apply as well. If it reports ones as missing that exist and + # that the user may read, the config's insecureLinks decides: populate + # fails with an error if it's null, syncs without them if it's false, and + # makes rsync open them like the files it transfers with --insecure-links + # if it's true, both with a warning. Sets insecure_links to rsync's extra + # argument. + insecureLinks = config: let + setting = config.insecureLinks; + in /* sh */ '' insecure_links= rsync_version=$(${rsync}/bin/rsync --version | ${coreutils}/bin/head -n 1) rsync_version=$(set -- $rsync_version; echo "$3") + # Whether a .rsync-filter file applies at all. + filtered=$(${findutils}/bin/find "$source_path" -name .rsync-filter -print -quit) + above=$(${coreutils}/bin/dirname "$source_path") + while test -z "$filtered"; do + if test -e "$above"/.rsync-filter; then + filtered=$above/.rsync-filter + elif test "$above" = / || test "$above" = .; then + break + fi + above=$(${coreutils}/bin/dirname "$above") + done if test "$(printf '%s\n' 3.5.0 "$rsync_version" | ${coreutils}/bin/sort -V | ${coreutils}/bin/head -n 1)" = 3.5.0 && - test -n "$(${findutils}/bin/find "$source_path" -name .rsync-filter -print -quit)" + test -n "$filtered" then - # The first existing .rsync-filter file rsync reports as missing. - ignored=$( - ${rsync}/bin/rsync --dry-run --debug=FILTER2 -F -rlptD \ - "$source_path" "$TMPDIR"/rsync-filter-check 2>&1 | - while read -r line; do - case $line in - "[sender] parse_filter_file("*") [not found]") - file=''${line#*parse_filter_file(} - file=''${file%,*,*} - if test -e "$file"; then - printf '%s\n' "$file" - break - fi - ;; - esac - done - ) + # A dry run of the same transfer, as far as filtering goes. + ${rsync}/bin/rsync --dry-run --debug=FILTER2 \ + ${rsyncFilterArgs config} \ + -rlptD \ + "$source_path" "$TMPDIR"/rsync-filter-check \ + > "$TMPDIR"/rsync-filter-check.log 2>&1 || : + # The first .rsync-filter file rsync reports as missing although it + # exists. One the user may not read isn't for rsync to read anyway. + parsed= + ignored= + while IFS= read -r line; do + case $line in + "[sender] parse_filter_file("*) + parsed=1 + ;; + esac + case $line in + "[sender] parse_filter_file("*") [not found]") + file=''${line#*parse_filter_file(} + file=''${file%,*,*} + # rsync prints bytes it takes for unprintable, which depends on + # the locale, as \#ooo. + escaped=$file + file= + while :; do + case $escaped in + *'\#'[0-7][0-7][0-7]*) + literal=''${escaped%%'\#'[0-7][0-7][0-7]*} + escaped=''${escaped#"$literal"'\#'} + byte=$(printf "\\$(printf %s "$escaped" | ${coreutils}/bin/cut -c1-3)x") + file=$file$literal''${byte%x} + escaped=''${escaped#???} + ;; + *) + file=$file$escaped + break + ;; + esac + done + if test -e "$file" && test -r "$file"; then + ignored=$file + break + fi + ;; + esac + done < "$TMPDIR"/rsync-filter-check.log + ${coreutils}/bin/rm "$TMPDIR"/rsync-filter-check.log + if test -z "$parsed"; then + >&2 printf 'warning: rsync %s doesn'"'"'t tell which .rsync-filter files it reads, so it may sync %s without some of them unnoticed.\n' \ + "$rsync_version" "$source_path" + fi if test -n "$ignored"; then ${if setting == null then /* sh */ '' >&2 printf 'error: rsync %s can'"'"'t read %s, so it would sync %s without it. This happens, e.g., if a directory on the way isn'"'"'t readable, or if a symlink on the way belongs to neither root nor the user running rsync. Set the source'"'"'s insecureLinks to true to make rsync read it anyway, or to false to sync without it.\n' \ |