1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
|
//! Shell code that runs on the target, where populate can't rely on more
//! than a POSIX shell and the usual tools. Values come as positional
//! parameters, never quoted into the code.
/// The directories below the sentinel, once that is a directory, which have
/// a directory for each source that needs one. Besides, there is backup,
/// which holds copies of the sources.
pub const STATE_KINDS: &str = "derived flake-inputs flake-uploads flakes patches";
/// Defines use_state SENTINEL NAME..., which prepares keeping state below
/// the sentinel for the sources named NAME. The sentinel only needs to
/// exist, so it may be a file, which becomes a directory. Populate changes
/// and removes what's in its state directories, and a symlink in place of
/// one would make it do that to whatever the link points to, so it refuses
/// to use those.
pub fn use_state() -> String {
format!(
r#"
refuse_link() {{
if test -L "$1"; then
>&2 printf 'error: %s is a symlink, but populate needs a directory of its own there\n' "$1"
exit 1
fi
}}
use_state() {{
sentinel=$1
shift
refuse_link "$sentinel"
refuse_link "$sentinel"/backup
for state_kind in {STATE_KINDS}; do
refuse_link "$sentinel"/"$state_kind"
for state_name; do
refuse_link "$sentinel"/"$state_kind"/"$state_name"
done
done
if test -f "$sentinel"; then
rm "$sentinel"
mkdir "$sentinel"
fi
}}
"#
)
}
/// Defines remove_tree PATH, which removes what's at PATH, with everything
/// below it, also what isn't writable. A symlink gets removed itself,
/// instead of followed. Only directories get made writable, which is what
/// removing what's in them needs: a file may be a hard link to one
/// elsewhere, which shares its mode.
pub const REMOVE_TREE: &str = r#"
remove_tree() {
if test -L "$1"; then
rm "$1"
elif test -e "$1"; then
find "$1" -type d -exec chmod u+w {} +
rm -fR "$1"
fi
}
"#;
/// Defines remove_link PATH NAME, which removes a symlink at PATH, saying
/// so, with NAME for PATH, because the link may be there on purpose, but
/// populate never follows it: what a source delivers would change or remove
/// what the link points to. remove_foreign_link PATH NAME does so unless
/// the link points into the store, like the links populate puts there
/// itself, and make_room_for_link PATH NAME makes room at PATH for a link
/// into the store, which Nix only puts in place of another link into the
/// store.
pub const LINKS: &str = r#"
remove_link() {
if test -L "$1"; then
>&2 printf "replacing symlink '%s', which points to '%s'...\n" \
"$2" "$(readlink "$1")"
rm "$1"
fi
}
remove_foreign_link() {
if test -L "$1"; then
# Resolved, so a link like /nix/store/../.. doesn't count.
case $(readlink -f "$1") in
/nix/store/*) ;;
*) remove_link "$1" "$2" ;;
esac
fi
}
make_room_for_link() {
remove_foreign_link "$1" "$2"
if ! test -L "$1" && test -e "$1"; then
rm -fR "$1"
fi
}
"#;
/// Defines remove_others DIR LOG NAME..., which removes the entries of DIR
/// that aren't named NAME. find, unlike a loop reading names, copes with
/// every name, e.g. ones that end in blanks or contain newlines. With LOG
/// empty, entries get removed like remove_tree does; for entries of the
/// target directory, LOG is the directory's name in the line printed for
/// each entry, and nothing gets made writable first. The names are
/// portable file names, which mean themselves as patterns of find.
pub const REMOVE_OTHERS: &str = r#"
remove_others() {
dir=$1
log=$2
shift 2
n=$#
while test "$n" -gt 0; do
set -- "$@" ! -name "$1"
shift
n=$((n - 1))
done
find "$dir" -mindepth 1 -maxdepth 1 "$@" -exec sh -c '
log=$1
shift
for path; do
if test -n "$log"; then
>&2 printf "pruning '\''%s/%s'\''...\n" "$log" "${path##*/}"
rm -fR "$path"
elif test -L "$path"; then
rm "$path"
else
find "$path" -type d -exec chmod u+w {\} +
rm -fR "$path"
fi
done
' sh "$log" {} +
}
"#;
/// check SENTINEL NAME DIR FORCE: fails unless the sentinel exists, creating
/// it and the target directory DIR first if FORCE isn't empty. NAME is the
/// sentinel's name in messages.
pub const CHECK: &str = r#"
sentinel=$1 name=$2 dir=$3 force=$4
# Before anything touches it, which would touch what it points to.
if test -L "$sentinel"; then
>&2 printf 'error: sentinel file is a symlink: %s\n' "$name"
exit 1
fi
if test -n "$force"; then
mkdir -vp "$dir" >&2
if ! test -e "$sentinel"; then
touch "$sentinel"
fi
fi
if ! test -e "$sentinel"; then
>&2 printf 'error: missing sentinel file: %s\n' "$name"
exit 1
fi
"#;
/// backup DIR NAME: copies the target directory DIR to the sentinel's
/// backup directory. NAME is the sentinel's name in messages.
pub fn backup() -> String {
use_state()
+ r#"
dir=$1 name=$2
use_state "$dir"/.populate
if ! test -d "$dir"/.populate; then
>&2 printf 'error: sentinel file is not a directory: %s\n' "$name"
exit 1
fi
rsync >&2 \
-aAXF \
--delete \
--exclude /.populate \
--link-dest="$dir" \
"$dir"/ \
"$dir"/.populate/backup/
"#
}
/// prune DIR LOG NAME...: removes the entries of the target directory DIR
/// that aren't sources named NAME, logging them with LOG for DIR, and
/// populate's state that only sources that are gone needed. The names
/// after -- are the ones of flake and flake-input sources, whose roots for
/// the garbage collector stay.
pub fn prune() -> String {
use_state()
+ REMOVE_TREE
+ REMOVE_OTHERS
+ r#"
dir=$1 log=$2
shift 2
names=
while test "$1" != --; do
names="$names $1"
shift
done
shift
use_state "$dir"/.populate
cd "$dir"
# The names are portable file names, so splitting them at blanks is fine.
# shellcheck disable=SC2086
remove_others . "$log" .populate $names
roots=.populate/flake-inputs
if test -d "$roots"; then
remove_others "$roots" "" "$@"
if test -z "$(ls -A "$roots")"; then
rmdir "$roots"
fi
fi
remove_tree .populate/flake-uploads
"#
}
/// break_hard_links PATH NAME: prepares PATH for rsync. rsync writes
/// through a symlink at the target path, e.g. one a symlink or flake source
/// left there, into what it points to. And rsync fixes the mode and time of
/// a file it takes for unchanged in place, which changes every hard link to
/// it, e.g. one elsewhere, or a backup's. So each file with other hard
/// links gets replaced by a copy of its own first.
pub fn break_hard_links() -> String {
LINKS.to_string()
+ r#"
remove_link "$1" "$2"
if test -d "$1"; then
find "$1" -type f -links +1 -exec sh -c '
for file; do
copy=$(mktemp "$file.XXXXXX")
cp -p "$file" "$copy"
mv -f "$copy" "$file"
done
' sh {} +
fi
"#
}
/// derivation PATH NAME TEXT: builds the Nix expression TEXT on the target,
/// with a link to the result at PATH.
pub fn derivation() -> String {
LINKS.to_string()
+ r#"
make_room_for_link "$1" "$2"
nix-build -E "$3" -o "$1" >&2
"#
}
/// symlink PATH TARGET: makes PATH a symlink to TARGET.
pub const SYMLINK: &str = r#"
ln -fnsT "$2" "$1"
"#;
/// Defines git_problems PATH, which prints why populate can't use the .git
/// directory of the Git work tree at PATH, or nothing. A .git directory
/// that git clone and populate made has no symlinks, no commondir file,
/// which names another repository, no linked work trees, and only the
/// configuration they write: the repository's format, the remote, branches,
/// and submodules, whose repositories have their work tree in the work
/// tree.
pub const GIT_PROBLEMS: &str = r#"
git_problems() (
path=$1
git_dir=$1/.git
links=$(find "$git_dir" -type l | head -n 3 | paste -s -d ' ' -)
if test -n "$links"; then
printf '%s contains symlinks: %s' "$git_dir" "$links"
exit
fi
common=$(find "$git_dir" -name commondir | head -n 1)
if test -n "$common"; then
printf '%s names another repository' "$common"
exit
fi
if test -e "$git_dir"/worktrees; then
printf '%s has linked work trees' "$git_dir"
exit
fi
find "$git_dir" -name config -type f | while IFS= read -r config; do
command git config --file "$config" --list --name-only |
while IFS= read -r key; do
case $key in
core.repositoryformatversion|core.filemode|core.bare|\
core.logallrefupdates|core.symlinks|core.ignorecase|\
core.precomposeunicode|\
extensions.objectformat|extensions.refstorage|\
remote.origin.url|remote.origin.fetch|\
branch.*.remote|branch.*.merge|\
submodule.active|submodule.*.url|submodule.*.active) ;;
core.worktree)
# Submodules' repositories have their work tree in the work tree.
worktree=$(command git config --file "$config" core.worktree)
dir=$(dirname "$config")
case $dir/$worktree in
"$git_dir"/modules/*)
resolved=$(cd "$dir" && cd "$worktree" 2>/dev/null && pwd -P) || :
case $resolved in
"$(cd "$path" && pwd -P)"/*) continue ;;
esac
;;
esac
printf '%s sets core.worktree to %s' "$config" "$worktree"
exit 1
;;
*)
printf '%s sets %s, which neither git clone nor populate does' "$config" "$key"
exit 1
;;
esac
done || exit 1
done || :
)
"#;
/// git PATH NAME URL REF SHALLOW FETCH_ALWAYS RECLONE EXCLUDE...: makes
/// PATH a Git work tree of the repository at URL, with REF checked out,
/// and everything removed that Git doesn't track, except what matches an
/// EXCLUDE pattern. SHALLOW, FETCH_ALWAYS, and RECLONE are flags, empty
/// for false. NAME is PATH's name in messages.
pub fn git() -> String {
LINKS.to_string()
+ GIT_PROBLEMS
+ r#"
path=$1 name=$2 url=$3 ref=$4 shallow=$5 fetch_always=$6 reclone=$7
shift 7
# Hooks and commands a repository's configuration names run with Git's
# commands, so populate's Git doesn't run any. The configuration gets
# checked as well, but Git also runs a hook it has itself.
git() {
command git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@"
}
# A target path that doesn't look like a Git work tree gets removed, e.g.
# when it had another type before. A work tree that a symlink points to,
# or whose .git is a symlink or a file naming a repository elsewhere, isn't
# populate's own: fetching, resetting, and cleaning would change what's
# elsewhere.
remove_link "$path" "$name"
if test -L "$path"/.git || ! test -d "$path"/.git; then
rm -fR "$path"
fi
# A .git directory can name places elsewhere too, which Git's commands
# change, or commands, which they run. Populate only uses a work tree
# whose .git directory has nothing like that, and else replaces it if
# reclone is set, or fails, telling why.
if test -e "$path"; then
if problems=$(git_problems "$path"); then :; else
exit 1
fi
if test -n "$problems"; then
if test -n "$reclone"; then
>&2 printf "replacing the Git work tree '%s', because %s...\n" \
"$name" "$problems"
rm -fR "$path"
else
>&2 printf 'error: populate doesn'"'"'t use the Git work tree %s, because %s. Fix that, remove the work tree, or set the source'"'"'s reclone to true to replace it by a fresh clone.\n' \
"$name" "$problems"
exit 1
fi
fi
fi
if ! test -e "$path"; then
if test -n "$shallow"; then
git init "$path"
else
git clone --recurse-submodules "$url" "$path"
fi
fi
cd "$path"
if ! origin=$(git config remote.origin.url); then
git remote add origin "$url"
elif test "$origin" != "$url"; then
git remote set-url origin "$url"
fi
# A commit, given by its hash or a prefix thereof, gets fetched only if the
# work tree lacks it, unless fetchAlways is set. A branch or tag gets
# fetched on every run, so the work tree follows it: resetting to its name
# would reset to a local branch, which fetching doesn't move.
fetch() {
if test -n "$shallow"; then
git fetch --depth=1 origin "$1"
else
git fetch origin "$1"
fi
}
fetch_commit() {
if test -n "$shallow"; then
fetch "$hash"
else
git fetch origin
fi
}
if printf %s "$ref" | grep -Eqx '[0-9a-f]{4,40}'; then
hash=$ref
if ! test "$(git log --format=%H -1)" = "$hash"; then
if test -n "$fetch_always"; then
fetch_commit
elif ! git log -1 "$hash" >/dev/null 2>&1; then
fetch_commit
fi
fi
else
fetch "$ref"
hash=$(git rev-parse --verify 'FETCH_HEAD^{commit}')
fi
# Also if HEAD is there already: rewriting a lock file in place, e.g.,
# changes the work tree, and stages the lock file if Nix created it.
git reset --hard "$hash" >&2
git submodule update --init --recursive
n=$#
while test "$n" -gt 0; do
set -- "$@" -e "$1"
shift
n=$((n - 1))
done
git clean -dfx "$@"
"#
}
/// Shell code running on the deploying machine, in the source's working
/// directory.
pub mod local {
/// pass_info DIR NAME: prints what tells whether the pass source of the
/// passwords below NAME in the store DIR changed: the last commits of
/// NAME, and of what symlinks therein point to, whose changes the
/// commits of NAME miss. Prints nothing if the store isn't a Git
/// repository.
pub const PASS_INFO: &str = r#"
dir=$1 name=$2
if test -e "$dir"/.git; then
printf '%s %s\n' "$name" "$(
git -C "$dir" log -1 --format=%H "$name"
find "$dir"/"$name" -type l -exec realpath {} + |
sort |
xargs -r -n 1 git -C "$dir" log -1 --format=%H
)"
fi
"#;
/// pass DIR NAME INFO: decrypts the passwords below NAME in the store
/// DIR into the working directory, with the times of their last
/// commits, and INFO, unless empty, in .pass_info.
pub const PASS: &str = r#"
dir=$1 name=$2 info=$3
umask 0077
chmod 0700 "$PWD"
prefix=$dir/$name
find "$prefix" -type f -follow ! -name .gpg-id |
while read -r gpg_path; do
rel_name=${gpg_path#"$prefix"}
rel_name=${rel_name%.gpg}
pass_date=$(
if test -e "$dir"/.git; then
git -C "$dir" log -1 --format=%aI "$gpg_path"
fi
)
pass_name=$name/$rel_name
pass_out=$PWD/$rel_name
mkdir -p "$(dirname "$pass_out")"
PASSWORD_STORE_DIR=$dir pass show "$pass_name" > "$pass_out"
if [ -n "$pass_date" ]; then
touch -d "$pass_date" "$pass_out"
fi
done
if test -n "$info"; then
printf '%s\n' "$info" > "$PWD"/.pass_info
fi
"#;
/// passage HOST: decrypts the secrets in $PASSAGE_DIR into the working
/// directory, with the times of their files. HOST is the first label
/// of the target's host, for the command a failure prints.
pub const PASSAGE: &str = r#"
host=$1
umask 0077
chmod 0700 "$PWD"
find "$PASSAGE_DIR" -type f -name \*.age -follow |
while read -r age_path; do
pass_name=${age_path#"$PASSAGE_DIR"/}
pass_name=${pass_name%.age}
pass_out=$PWD/$pass_name
mkdir -p "$(dirname "$pass_out")"
if ! passage show "$pass_name" > "$pass_out" 2>&1; then
echo ++ secrets "$host" show "$pass_name" >&2
jq -Rr 'select(test("^[^[]"))' "$pass_out" >&2
exit 1
fi
touch -r "$age_path" "$pass_out"
done
"#;
}
/// pass_info PATH: prints the .pass_info the pass source at PATH got last.
pub const PASS_INFO: &str = r#"
cat "$1"/.pass_info || :
"#;
|