diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d9d985e75..fba465592 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -197,6 +197,16 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Verify a failed image pull preserves deployment metadata run: bash deploy/host/tests/upgrade-failed-pull.sh + - name: Verify the Compose override registry + run: bash deploy/host/tests/compose-overrides.sh + - name: Verify nonstandard Docker binary support + run: bash deploy/host/tests/compose-docker-path.sh + - name: Verify failed backup restart cleanup + run: bash deploy/host/tests/backup-failed-restart.sh + - name: Verify failed restore cleanup + run: bash deploy/host/tests/restore-failed-cleanup.sh + - name: Verify upgrade refreshes the CLI and systemd unit + run: bash deploy/host/tests/upgrade-refresh.sh - name: Restore an encrypted bundle onto empty volumes run: bash deploy/host/tests/backup-restore.integration.sh diff --git a/SELF_HOSTING.md b/SELF_HOSTING.md index d6c01e8df..d53069220 100644 --- a/SELF_HOSTING.md +++ b/SELF_HOSTING.md @@ -147,6 +147,31 @@ mode only when you know public DNS is not ready yet. The selected mode is retained in `/opt/roomote/.env`, so installer reruns and `roomote upgrade` preserve it. +### Compose overrides + +To customize the stack beyond what `.env` covers (for example a Caddy image +built with a DNS-provider plugin), put your changes in a Compose override file +next to the managed base file and register it: + +```sh +sudo roomote override add docker-compose.caddy-dns.yml +``` + +The file must already exist in `/opt/roomote` as a plain `.yml`/`.yaml` file +name. Registered overrides are recorded in `COMPOSE_FILE` in +`/opt/roomote/.env` and layered after `docker-compose.prod.yml` in the listed +order, for every `roomote` command, systemd start, and boot. They are included +in backup bundles, restored by `roomote restore`, and preserved across +`roomote upgrade` (which refreshes only the managed base file and Caddyfile). +`roomote override list` shows the merge order and `roomote override remove` +unregisters a file without deleting it. + +Never edit `docker-compose.prod.yml` or the managed Caddyfile directly: both +are replaced on every upgrade. One caveat for overrides that swap in custom +images: backup bundles record image identities for the managed stack only, so +a locally built override image must be rebuilt or re-pulled by you after a +fresh-host restore. + ### Cloudflare Tunnel Cloudflare Tunnel works with the supported `internal` TLS mode: Cloudflare diff --git a/apps/docs/self-hosting.mdx b/apps/docs/self-hosting.mdx index cce01adf1..96a987023 100644 --- a/apps/docs/self-hosting.mdx +++ b/apps/docs/self-hosting.mdx @@ -91,8 +91,28 @@ roomote upgrade # pull and roll out newer images roomote rollback # return to the release before the last upgrade roomote backup # create an encrypted deployment recovery bundle roomote logs # tail service logs +roomote override # register operator Compose override files ``` +### Customizing the stack with Compose overrides + +`docker-compose.prod.yml` and the managed Caddyfile are replaced on every +upgrade, so never edit them directly. Put customizations (for example a Caddy +image built with a DNS-provider plugin) in an override file in `/opt/roomote` +and register it: + +```sh +sudo roomote override add docker-compose.caddy-dns.yml +``` + +Registered overrides are layered after the base file, in registration order, +for every `roomote` command, systemd restart, and boot. They ride along in +backup bundles, come back with `roomote restore`, and survive `roomote +upgrade`. `roomote override list` shows the merge order; `roomote override +remove` unregisters a file without deleting it. Images referenced only by +override files are not covered by backup image identities, so keep locally +built override images rebuildable on a replacement host. + `roomote backup` prompts for a passphrase and writes a versioned `.roomote` bundle under `/opt/roomote/backups`. The bundle contains PostgreSQL, the deployment configuration and encryption/signing keys, local MinIO artifacts, @@ -136,7 +156,9 @@ empty PostgreSQL/MinIO/Redis volumes, and starts the recorded Roomote release. that also rewinds data. Use `roomote upgrade` instead of updating application image references alone. -The command refreshes the release's Compose and Caddy configuration together; +The command refreshes the release's Compose and Caddy configuration, the +`roomote` CLI itself, and the systemd unit together (operator override files +and their registration are left untouched); mixing newer application images with an older Caddyfile can leave routes used by the new controller unavailable until the deployment configuration is also updated. diff --git a/deploy/ci/validate-deployment-artifacts.mjs b/deploy/ci/validate-deployment-artifacts.mjs index 70109bfe9..363961eb0 100644 --- a/deploy/ci/validate-deployment-artifacts.mjs +++ b/deploy/ci/validate-deployment-artifacts.mjs @@ -17,6 +17,8 @@ const read = (path) => readFileSync(join(root, path), 'utf8'); const catalog = JSON.parse(read('deploy/deployment-catalog.json')); const installer = read('deploy/install.sh'); const deployer = read('deploy/scripts/deploy.sh'); +const upgrader = read('deploy/scripts/upgrade.sh'); +const hostCli = read('deploy/host/roomote'); const productionEnvExample = read('.env.production.example'); function fail(message) { @@ -27,6 +29,49 @@ function assert(condition, message) { if (!condition) fail(message); } +// The host CLI is the single owner of the Compose invocation (COMPOSE_FILE +// registry in .env) and of the systemd unit. Every other surface must route +// through it so operator overrides can never be dropped by a sibling path. +assert( + hostCli.includes('read_env_value COMPOSE_FILE') && + hostCli.includes('compose_args+=(-f "$install_root/$entry")'), + 'host CLI: compose invocations must be built from the COMPOSE_FILE registry', +); +assert( + hostCli.includes('ExecStart=$cli_path up') && + hostCli.includes('ExecStop=$cli_path down') && + hostCli.includes('refresh_host_cli "$repo" "$fetch_ref"'), + 'host CLI: it must own the systemd unit and refresh itself during upgrades', +); +assert( + installer.includes('/usr/local/bin/roomote sync-unit') && + installer.includes('/usr/local/bin/roomote up') && + !installer.includes('docker compose --env-file'), + 'installer: systemd unit and stack start must go through the host CLI', +); +for (const [name, script] of [ + ['deploy.sh', deployer], + ['upgrade.sh', upgrader], +]) { + assert( + script.includes('/usr/local/bin/roomote sync-unit') && + script.includes('/usr/local/bin/roomote up') && + script.includes('/usr/local/bin/roomote docker pull') && + !script.includes('docker compose --env-file') && + !/^docker /m.test(script), + `managed ${name}: remote docker and compose operations must go through the host CLI`, + ); +} +assert( + deployer.includes('for key in COMPOSE_FILE ROOMOTE_DOCKER_BIN'), + 'managed deploy.sh: redeploys must carry forward host-owned .env state (override registry, Docker path)', +); +assert( + hostCli.includes('compose-overrides') && + hostCli.includes("backup_services_stopped='false'"), + 'host CLI: backups must stage override files and keep trap state global', +); + assert( installer.includes('preview_domain="$domain"') && installer.includes( @@ -610,7 +655,12 @@ for (const script of [ 'deploy/ci/deployment-smoke.sh', 'deploy/ci/upgrade-compatibility.sh', 'deploy/host/tests/backup-restore.integration.sh', + 'deploy/host/tests/backup-failed-restart.sh', + 'deploy/host/tests/compose-docker-path.sh', + 'deploy/host/tests/compose-overrides.sh', + 'deploy/host/tests/restore-failed-cleanup.sh', 'deploy/host/tests/upgrade-failed-pull.sh', + 'deploy/host/tests/upgrade-refresh.sh', '.docker/gbrain/entrypoint.sh', ]) { execFileSync('bash', ['-n', join(root, script)], { stdio: 'pipe' }); diff --git a/deploy/host/roomote b/deploy/host/roomote index 0ebef393f..66c48796c 100755 --- a/deploy/host/roomote +++ b/deploy/host/roomote @@ -8,6 +8,9 @@ set -euo pipefail install_root="${ROOMOTE_INSTALL_ROOT:-/opt/roomote}" env_file="$install_root/.env" compose_file="$install_root/docker-compose.prod.yml" +compose_base_name="docker-compose.prod.yml" +cli_path="${ROOMOTE_CLI_PATH:-/usr/local/bin/roomote}" +systemd_dir="${ROOMOTE_SYSTEMD_DIR:-/etc/systemd/system}" usage() { cat <<'EOF' @@ -31,6 +34,17 @@ Commands: restart [service...] Restart services (default: all) down Stop the stack up Start the stack + override [file] + Manage operator Compose override files. Overrides are + recorded in COMPOSE_FILE in /opt/roomote/.env, layered + after the managed base file in the listed order, and + preserved across upgrades, backups, and restarts + compose [args...] Run docker compose against the full deployment file + list (base file plus registered overrides) + docker [args...] Run the deployment's recorded Docker binary + (ROOMOTE_DOCKER_BIN in .env) + sync-unit (Re)write the roomote-compose systemd unit so boots + and systemd restarts go through this CLI EOF } @@ -49,8 +63,62 @@ fi [ -f "$env_file" ] || die "$env_file not found; is Roomote installed on this host?" [ -f "$compose_file" ] || die "$compose_file not found; re-run the installer" +# All Docker invocations go through docker_cmd so nonstandard Docker installs +# (snap, manual binary) recorded in .env as ROOMOTE_DOCKER_BIN work everywhere, +# including under systemd's minimal PATH. docker_bin is resolved after the +# function definitions, right before command dispatch. +docker_cmd() { + "$docker_bin" "$@" +} + +# The deployment's Compose file list is COMPOSE_FILE in .env (colon-separated, +# relative to $install_root, base file first) -- the same variable docker +# compose itself honors, so a bare `docker compose` run from /opt/roomote sees +# the same stack. Managed by `roomote override`; absent means base file only. +# Populates compose_files_list, validating each entry. Runs in the calling +# shell (not a pipeline) so `die` on a bad entry stops the command. +compose_files_list=() +load_compose_files() { + local raw entry + local -a parts=() + + compose_files_list=() + raw="$(read_env_value COMPOSE_FILE)" + [ -n "$raw" ] || raw="$compose_base_name" + IFS=':' read -r -a parts <<<"$raw" + + for entry in "${parts[@]}"; do + [ -n "$entry" ] || continue + case "$entry" in + /* | *..* | */*) die "COMPOSE_FILE entry '$entry' in $env_file must be a plain file name inside $install_root" ;; + esac + compose_files_list+=("$entry") + done + + [ "${#compose_files_list[@]}" -gt 0 ] || die "COMPOSE_FILE in $env_file is empty" + [ "${compose_files_list[0]}" = "$compose_base_name" ] || + die "COMPOSE_FILE in $env_file must list the managed base file $compose_base_name first (found '${compose_files_list[0]}')" +} + compose() { - docker compose --env-file "$env_file" -f "$compose_file" "$@" + local entry + local -a compose_args=(compose --env-file "$env_file") + + load_compose_files + for entry in "${compose_files_list[@]}"; do + [ -f "$install_root/$entry" ] || + die "Compose file $install_root/$entry (from COMPOSE_FILE in $env_file) not found; restore it or run 'roomote override remove $entry'" + compose_args+=(-f "$install_root/$entry") + done + + docker_cmd "${compose_args[@]}" "$@" +} + +# Base-file-only variant for the pieces of the deployment Roomote itself +# manages (e.g. backup image-identity collection, where operator override +# images are the operator's recovery responsibility). +compose_base() { + docker_cmd compose --env-file "$env_file" -f "$compose_file" "$@" } backup_usage() { @@ -175,7 +243,7 @@ container_volume_name() { container_id="$(compose ps -aq "$service")" [ -n "$container_id" ] || die "could not find the $service container" - volume="$(docker inspect --format "{{range .Mounts}}{{if eq .Destination \"$destination\"}}{{.Name}}{{end}}{{end}}" "$container_id")" + volume="$(docker_cmd inspect --format "{{range .Mounts}}{{if eq .Destination \"$destination\"}}{{.Name}}{{end}}{{end}}" "$container_id")" [ -n "$volume" ] || die "could not find the $service volume mounted at $destination" printf '%s' "$volume" } @@ -185,7 +253,7 @@ archive_volume() { local staging_dir="$2" local archive_name="$3" - docker run --rm \ + docker_cmd run --rm \ --volume "$volume:/source:ro" \ --volume "$staging_dir:/backup" \ --env "ARCHIVE_NAME=$archive_name" \ @@ -201,12 +269,12 @@ restore_volume() { local archive_name="${3:-}" if [ -n "$archive_name" ]; then - docker run --rm \ + docker_cmd run --rm \ --volume "$volume:/target" \ --volume "$staging_dir:/backup:ro" \ redis:7-alpine sh -c "rm -rf /target/* /target/.[!.]* /target/..?*; tar -C /target -xf /backup/$archive_name" else - docker run --rm \ + docker_cmd run --rm \ --volume "$volume:/target" \ redis:7-alpine sh -c 'rm -rf /target/* /target/.[!.]* /target/..?*' fi @@ -217,10 +285,10 @@ postgres_command() { shift || true if [ -n "$input_file" ]; then - docker run --rm --network "$(compose_network_name)" --env-file "$env_file" -i \ + docker_cmd run --rm --network "$(compose_network_name)" --env-file "$env_file" -i \ pgvector/pgvector:0.8.1-pg17-trixie@sha256:137f044b0efe3d57f39b972b9b53641b1f2045b99d879e298bbf514a25787dcf sh -c "$*" <"$input_file" else - docker run --rm --network "$(compose_network_name)" --env-file "$env_file" \ + docker_cmd run --rm --network "$(compose_network_name)" --env-file "$env_file" \ pgvector/pgvector:0.8.1-pg17-trixie@sha256:137f044b0efe3d57f39b972b9b53641b1f2045b99d879e298bbf514a25787dcf sh -c "$*" fi } @@ -325,7 +393,7 @@ prune_roomote_images() { keep_file="$(mktemp)" || return images_file="$(mktemp)" || return - docker image ls --format '{{.CreatedAt}}|{{.Repository}}|{{.Tag}}' | + docker_cmd image ls --format '{{.CreatedAt}}|{{.Repository}}|{{.Tag}}' | awk -F'|' -v prefix="$repo_prefix" 'index($2, prefix) == 1 && $3 != "" { print $1 "|" $3 }' | sort -t'|' -k1,1r | awk -F'|' '!seen[$2]++ { print $2 }' >"$tags_file" || return @@ -354,7 +422,7 @@ prune_roomote_images() { cat "${keep_file}.dedup" >"$keep_file" || return rm -f "${keep_file}.dedup" - docker image ls --format '{{.Repository}}:{{.Tag}}|{{.Repository}}|{{.Tag}}' | + docker_cmd image ls --format '{{.Repository}}:{{.Tag}}|{{.Repository}}|{{.Tag}}' | awk -F'|' -v prefix="$repo_prefix" 'index($2, prefix) == 1 && $3 != "" { print }' >"$images_file" || return while IFS='|' read -r image repo tag; do @@ -363,7 +431,7 @@ prune_roomote_images() { continue fi - if docker image rm "$image" >/dev/null 2>&1; then + if docker_cmd image rm "$image" >/dev/null 2>&1; then removed=$((removed + 1)) printf 'Removed old Roomote image %s\n' "$image" else @@ -372,7 +440,7 @@ prune_roomote_images() { fi done <"$images_file" - docker image prune -f >/dev/null 2>&1 || true + docker_cmd image prune -f >/dev/null 2>&1 || true kept_count="$(awk 'NF { count++ } END { print count + 0 }' "$keep_file")" || return kept_tags="$(tr '\n' ' ' <"$keep_file" | sed 's/[[:space:]]*$//')" || return @@ -404,6 +472,152 @@ cmd_setup_url() { fi } +override_usage() { + cat <<'EOF' +usage: roomote override [file] + + add Register an operator Compose override. The file must already + exist in /opt/roomote as a plain .yml/.yaml file name (for + example docker-compose.caddy-dns.yml). The merged + configuration is validated before the override is recorded. + remove Unregister an override (the file itself is left in place) + list Print the deployment's Compose file list in merge order + +Overrides are stored in COMPOSE_FILE in /opt/roomote/.env, layered after the +managed base file in the listed order, included in backup bundles, and +preserved across upgrades and systemd restarts. Images referenced only by +override files stay operator-managed: they are not covered by backup image +identities, so make them pullable (or rebuildable) on a restored host. +EOF +} + +serialize_compose_files() { + local IFS=':' + printf '%s' "${compose_files_list[*]}" +} + +cmd_override() { + local action="${1:-}" + local name="${2:-}" + local entry + local -a candidate=() + + case "$action" in + list) + load_compose_files + for entry in "${compose_files_list[@]}"; do + if [ "$entry" = "$compose_base_name" ]; then + printf '%s (managed base file)\n' "$entry" + else + printf '%s\n' "$entry" + fi + done + return + ;; + add) + [ -n "$name" ] || { override_usage >&2; die "usage: roomote override add "; } + name="${name##*/}" + case "$name" in + *.yml | *.yaml) ;; + *) die "override file name must end in .yml or .yaml: $name" ;; + esac + case "$name" in + *:* | *..*) die "override file name must be a plain file name: $name" ;; + esac + [ "$name" != "$compose_base_name" ] || die "$compose_base_name is the managed base file, not an override" + [ -f "$install_root/$name" ] || die "$install_root/$name not found; place the override file there first" + + load_compose_files + for entry in "${compose_files_list[@]}"; do + [ "$entry" != "$name" ] || die "$name is already registered" + done + + # Validate the merged configuration before recording anything. + candidate=(compose --env-file "$env_file") + for entry in "${compose_files_list[@]}"; do + candidate+=(-f "$install_root/$entry") + done + candidate+=(-f "$install_root/$name") + docker_cmd "${candidate[@]}" config >/dev/null || + die "the merged Compose configuration with $name is invalid; fix the override and retry" + + compose_files_list+=("$name") + set_env_value COMPOSE_FILE "$(serialize_compose_files)" + log "Registered Compose override $name (COMPOSE_FILE=$(serialize_compose_files))" + log "The override now applies to every roomote up/upgrade/backup and to systemd starts" + ;; + remove) + [ -n "$name" ] || { override_usage >&2; die "usage: roomote override remove "; } + name="${name##*/}" + [ "$name" != "$compose_base_name" ] || die "the managed base file cannot be removed" + + load_compose_files + local -a remaining=() + local found='false' + for entry in "${compose_files_list[@]}"; do + if [ "$entry" = "$name" ]; then + found='true' + else + remaining+=("$entry") + fi + done + [ "$found" = 'true' ] || die "$name is not a registered override (see roomote override list)" + compose_files_list=("${remaining[@]}") + set_env_value COMPOSE_FILE "$(serialize_compose_files)" + log "Unregistered Compose override $name; the file itself was left in $install_root" + ;; + --help | -h | help | '') + override_usage + ;; + *) + override_usage >&2 + die "unknown override action: $action" + ;; + esac +} + +# The systemd unit delegates to this CLI so boots and `systemctl restart` +# use the same override-aware Compose invocation as every other operation. +# Rewritten by install.sh and by every `roomote upgrade` (sync-unit), so unit +# fixes reach existing deployments without an installer rerun. +sync_systemd_unit() { + cat >"$systemd_dir/roomote-compose.service" </dev/null 2>&1; then + systemctl daemon-reload >/dev/null 2>&1 || true + systemctl enable roomote-compose.service >/dev/null 2>&1 || true + fi +} + +cmd_sync_unit() { + [ "$#" -eq 0 ] || die "usage: roomote sync-unit" + # The unit delegates to this CLI, which resolves Docker from .env because + # systemd's minimal PATH misses snap/manual installs. Record the path before + # a boot depends on it: managed deploys and upgrades call sync-unit without + # the installer, so this is where the guarantee must live. + [ -n "$(read_env_value ROOMOTE_DOCKER_BIN)" ] || + set_env_value ROOMOTE_DOCKER_BIN "$(command -v docker || printf 'docker')" + sync_systemd_unit + log "Wrote $systemd_dir/roomote-compose.service (ExecStart=$cli_path up)" +} + resolve_latest_version() { local repo="$1" local api="https://api.github.com/repos/$repo" @@ -475,6 +689,15 @@ cleanup_upgrade() { else rm -f "$install_root/caddy/Caddyfile" fi + if [ -f "$upgrade_rollback_dir/roomote-cli" ]; then + install -m 755 "$upgrade_rollback_dir/roomote-cli" "$cli_path" + fi + if [ -f "$upgrade_rollback_dir/roomote-compose.service" ]; then + install -m 644 "$upgrade_rollback_dir/roomote-compose.service" "$systemd_dir/roomote-compose.service" + if command -v systemctl >/dev/null 2>&1; then + systemctl daemon-reload >/dev/null 2>&1 || true + fi + fi if [ "${upgrade_controller_stopped:-false}" = 'true' ]; then compose start controller || true fi @@ -483,6 +706,23 @@ cleanup_upgrade() { exit "$status" } +# Refreshes /usr/local/bin/roomote from the release being upgraded to, so CLI +# and systemd-unit fixes reach existing deployments through `roomote upgrade` +# rather than only through installer reruns. The download lands in a temp file +# and moves into place atomically: the running bash keeps executing the old +# inode, and a truncated download is rejected before it can replace the CLI. +refresh_host_cli() { + local repo="$1" + local fetch_ref="$2" + local tmp_cli + + tmp_cli="$(mktemp "$cli_path.XXXXXX")" + fetch_deploy_file "$repo" "$fetch_ref" deploy/host/roomote "$tmp_cli" + bash -n "$tmp_cli" || { rm -f "$tmp_cli"; die "downloaded host CLI failed a syntax check; keeping the current CLI"; } + chmod 755 "$tmp_cli" + mv -f "$tmp_cli" "$cli_path" +} + cmd_upgrade() { local version='' local skip_backup='false' @@ -546,6 +786,12 @@ cmd_upgrade() { if [ -f "$install_root/caddy/Caddyfile" ]; then install -m 600 "$install_root/caddy/Caddyfile" "$upgrade_rollback_dir/Caddyfile" fi + if [ -f "$cli_path" ]; then + install -m 755 "$cli_path" "$upgrade_rollback_dir/roomote-cli" + fi + if [ -f "$systemd_dir/roomote-compose.service" ]; then + install -m 644 "$systemd_dir/roomote-compose.service" "$upgrade_rollback_dir/roomote-compose.service" + fi image_registry="$(read_env_value IMAGE_REGISTRY)" image_namespace="$(read_env_value IMAGE_NAMESPACE)" @@ -565,6 +811,17 @@ cmd_upgrade() { log "Refreshing deployment files for $version" fetch_deploy_file "$repo" "$fetch_ref" deploy/compose/docker-compose.prod.yml "$compose_file" fetch_deploy_file "$repo" "$fetch_ref" deploy/caddy/Caddyfile "$install_root/caddy/Caddyfile" + log "Refreshing the roomote host CLI and systemd unit for $version" + refresh_host_cli "$repo" "$fetch_ref" + # Run sync-unit from the freshly installed CLI so the unit matches the new + # release even when its format changed after this (still-running) version. + "$cli_path" sync-unit >/dev/null + + # Older installs predate the COMPOSE_FILE registry and the persisted Docker + # path; seed both so the rest of this upgrade and the rewritten systemd unit + # behave identically on upgraded and freshly installed hosts. + [ -n "$(read_env_value COMPOSE_FILE)" ] || set_env_value COMPOSE_FILE "$compose_base_name" + [ -n "$(read_env_value ROOMOTE_DOCKER_BIN)" ] || set_env_value ROOMOTE_DOCKER_BIN "$(command -v docker || printf 'docker')" set_env_value ROOMOTE_VERSION "$version" # Recorded for `roomote rollback`; skipped for same-version re-runs so a @@ -603,7 +860,7 @@ cmd_upgrade() { compose stop controller || true upgrade_controller_stopped='true' log "Pulling images for $version" - docker pull "$worker_image" + docker_cmd pull "$worker_image" compose pull # Migrations run before any running service is replaced. Drizzle applies # all pending migrations in a single transaction, so a failure here rolls @@ -616,7 +873,6 @@ cmd_upgrade() { trap - EXIT log "Starting Roomote $version" compose up -d --wait --wait-timeout 600 - systemctl enable roomote-compose.service >/dev/null 2>&1 || true log "Pruning old Roomote images" if ! prune_roomote_images; then log "Warning: image retention failed; upgrade is still healthy" @@ -641,11 +897,11 @@ stop_task_workers() { local worker_ids worker_network worker_network="$(read_env_value DOCKER_WORKER_NETWORK)" worker_network="${worker_network:-roomote_worker}" - worker_ids="$(docker ps --filter "network=$worker_network" --format '{{.ID}} {{.Names}}' | awk '$2 ~ /^roomote-worker-/ { print $1 }')" + worker_ids="$(docker_cmd ps --filter "network=$worker_network" --format '{{.ID}} {{.Names}}' | awk '$2 ~ /^roomote-worker-/ { print $1 }')" if [ -n "$worker_ids" ]; then log "Stopping active task worker containers for a consistent backup" # shellcheck disable=SC2086 - docker stop --timeout 30 $worker_ids + docker_cmd stop --timeout 30 $worker_ids fi } @@ -654,16 +910,20 @@ collect_image_identities() { local references_file ref digest image_id worker_image references_file="$(mktemp)" - compose config --images >"$references_file" + # Base file only, on purpose: images referenced solely by operator override + # files are operator-managed and may be locally built with no repository + # digest, which must not block backups. `roomote override add` documents + # that override images are the operator's recovery responsibility. + compose_base config --images >"$references_file" worker_image="$(read_env_value DOCKER_WORKER_IMAGE)" [ -z "$worker_image" ] || printf '%s\n' "$worker_image" >>"$references_file" : >"$output_file" while IFS= read -r ref; do [ -n "$ref" ] || continue - docker image inspect "$ref" >/dev/null 2>&1 || die "image $ref is not present; pull it before creating a recovery backup" - digest="$(docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$ref" | head -n 1)" - image_id="$(docker image inspect --format '{{.Id}}' "$ref")" + docker_cmd image inspect "$ref" >/dev/null 2>&1 || die "image $ref is not present; pull it before creating a recovery backup" + digest="$(docker_cmd image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$ref" | head -n 1)" + image_id="$(docker_cmd image inspect --format '{{.Id}}' "$ref")" [ -n "$digest" ] || die "image $ref has no repository digest and cannot be recovered on a fresh host" printf '%s|%s|%s\n' "$ref" "$digest" "$image_id" >>"$output_file" done < <(sort -u "$references_file") @@ -678,23 +938,36 @@ restore_image_identities() { [ -n "$ref" ] || continue [ -n "$digest" ] || die "backup has no restorable digest for image $ref ($image_id)" log "Restoring image identity $digest" - docker pull "$digest" + docker_cmd pull "$digest" case "$ref" in *@*) ;; - *) docker tag "$digest" "$ref" ;; + *) docker_cmd tag "$digest" "$ref" ;; esac done <"$identities_file" } +# Runs from the EXIT trap. Reads only backup_* globals: EXIT traps fire after +# bash pops function locals on a set -e failure inside cmd_backup, so any +# function-local state would be unbound here under set -u. +cleanup_backup() { + if [ "$backup_services_stopped" = 'true' ]; then + log "Restarting Roomote after interrupted backup" + compose up -d --wait --wait-timeout 600 || true + fi + rm -rf "$backup_staging_dir" + if [ "$backup_generated_passphrase" = 'true' ]; then + rm -f "$backup_passphrase_file" + fi +} + cmd_backup() { local include_redis='false' local passphrase_arg='' local output_file='' - local generated_passphrase='false' - local passphrase_file staging_dir schema_version roomote_version + local entry + local schema_version roomote_version local storage_mode storage_included s3_endpoint s3_bucket local minio_volume redis_volume created_at - local services_stopped='false' while [ "$#" -gt 0 ]; do case "$1" in @@ -732,44 +1005,49 @@ cmd_backup() { mkdir -p "$(dirname -- "$output_file")" [ ! -e "$output_file" ] || die "backup output already exists: $output_file" + # Cleanup state for the EXIT trap. Deliberately not locals (see + # cleanup_backup); assigned before the trap is installed so the trap can + # always read them. + backup_generated_passphrase='false' if [ -z "$passphrase_arg" ]; then - generated_passphrase='true' + backup_generated_passphrase='true' fi - passphrase_file="$(make_passphrase_file "$passphrase_arg" confirm)" - staging_dir="$(mktemp -d "$install_root/backups/.backup-staging.XXXXXX")" - chmod 700 "$staging_dir" - - cleanup_backup() { - if [ "$services_stopped" = 'true' ]; then - log "Restarting Roomote after interrupted backup" - compose up -d --wait --wait-timeout 600 || true - fi - rm -rf "$staging_dir" - if [ "$generated_passphrase" = 'true' ]; then - rm -f "$passphrase_file" - fi - } + backup_services_stopped='false' + backup_passphrase_file="$(make_passphrase_file "$passphrase_arg" confirm)" + backup_staging_dir="$(mktemp -d "$install_root/backups/.backup-staging.XXXXXX")" + chmod 700 "$backup_staging_dir" trap cleanup_backup EXIT - mkdir -p "$staging_dir/config/caddy" - install -m 600 "$env_file" "$staging_dir/config/.env" - install -m 600 "$compose_file" "$staging_dir/config/docker-compose.prod.yml" + mkdir -p "$backup_staging_dir/config/caddy" + install -m 600 "$env_file" "$backup_staging_dir/config/.env" + install -m 600 "$compose_file" "$backup_staging_dir/config/docker-compose.prod.yml" + # Operator Compose overrides are part of the deployment configuration: + # without them a disaster-recovery restore would come up with the base + # stack only (e.g. Caddy missing its DNS-challenge config). + load_compose_files + for entry in "${compose_files_list[@]}"; do + [ "$entry" != "$compose_base_name" ] || continue + [ -f "$install_root/$entry" ] || + die "Compose override $install_root/$entry (from COMPOSE_FILE in $env_file) not found; restore it or run 'roomote override remove $entry'" + mkdir -p "$backup_staging_dir/config/compose-overrides" + install -m 600 "$install_root/$entry" "$backup_staging_dir/config/compose-overrides/$entry" + done if [ -f "$install_root/caddy/Caddyfile" ]; then - install -m 600 "$install_root/caddy/Caddyfile" "$staging_dir/config/caddy/Caddyfile" + install -m 600 "$install_root/caddy/Caddyfile" "$backup_staging_dir/config/caddy/Caddyfile" fi if [ -f "$install_root/deployment.env" ]; then - install -m 600 "$install_root/deployment.env" "$staging_dir/config/deployment.env" + install -m 600 "$install_root/deployment.env" "$backup_staging_dir/config/deployment.env" fi - collect_image_identities "$staging_dir/images.txt" + collect_image_identities "$backup_staging_dir/images.txt" log "Quiescing Roomote application writers" compose stop web api controller bullmq preview-proxy stop_task_workers - services_stopped='true' + backup_services_stopped='true' log "Dumping PostgreSQL" postgres_command '' 'pg_dump --clean --if-exists --no-owner --no-privileges "$DATABASE_URL"' \ - >"$staging_dir/postgres.sql" + >"$backup_staging_dir/postgres.sql" schema_version="$(postgres_command '' 'psql "$DATABASE_URL" -Atqc "SELECT hash FROM drizzle.__drizzle_migrations ORDER BY created_at DESC LIMIT 1"' 2>/dev/null || true)" schema_version="${schema_version:-unknown}" @@ -783,7 +1061,7 @@ cmd_backup() { log "Snapshotting the local MinIO volume" compose stop minio minio_volume="$(container_volume_name minio /data)" - archive_volume "$minio_volume" "$staging_dir" minio-data.tar + archive_volume "$minio_volume" "$backup_staging_dir" minio-data.tar else storage_mode='external-bucket' storage_included='false' @@ -795,12 +1073,12 @@ cmd_backup() { compose exec -T redis redis-cli SAVE >/dev/null compose stop redis redis_volume="$(container_volume_name redis /data)" - archive_volume "$redis_volume" "$staging_dir" redis-data.tar + archive_volume "$redis_volume" "$backup_staging_dir" redis-data.tar fi roomote_version="$(read_env_value ROOMOTE_VERSION)" created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - cat >"$staging_dir/manifest.json" <"$backup_staging_dir/manifest.json" <SHA256SUMS ) log "Encrypting backup bundle" - tar -C "$staging_dir" -czf - . | + tar -C "$backup_staging_dir" -czf - . | openssl enc -aes-256-cbc -salt -pbkdf2 -iter 200000 -md sha256 \ - -pass "file:$passphrase_file" -out "$output_file" + -pass "file:$backup_passphrase_file" -out "$output_file" chmod 600 "$output_file" log "Restarting Roomote" compose up -d --wait --wait-timeout 600 - services_stopped='false' - rm -rf "$staging_dir" - if [ "$generated_passphrase" = 'true' ]; then - rm -f "$passphrase_file" - fi + backup_services_stopped='false' trap - EXIT + cleanup_backup log "Encrypted deployment backup written to $output_file" } @@ -878,12 +1153,20 @@ validate_volume_archive() { die "volume snapshot contains unsupported links or special files" } +# Runs from the EXIT trap. Reads only restore_* globals for the same +# function-scope reason as cleanup_backup. +cleanup_restore() { + rm -rf "$restore_staging_dir" + if [ "$restore_generated_passphrase" = 'true' ]; then + rm -f "$restore_passphrase_file" + fi +} + cmd_restore() { local backup_file='' local confirmed='false' local passphrase_arg='' - local generated_passphrase='false' - local passphrase_file staging_dir encrypted_tar + local encrypted_tar entry restored_docker_bin local redis_included='false' local local_minio_included='false' local redis_volume minio_volume @@ -920,93 +1203,106 @@ cmd_restore() { command -v openssl >/dev/null 2>&1 || die "openssl is required" command -v sha256sum >/dev/null 2>&1 || die "sha256sum is required" + # Cleanup state for the EXIT trap. Deliberately not locals (see + # cleanup_restore); assigned before the trap is installed. + restore_generated_passphrase='false' if [ -z "$passphrase_arg" ]; then - generated_passphrase='true' + restore_generated_passphrase='true' fi - passphrase_file="$(make_passphrase_file "$passphrase_arg" read)" - staging_dir="$(mktemp -d "$install_root/backups/.restore-staging.XXXXXX")" - chmod 700 "$staging_dir" - encrypted_tar="$staging_dir/bundle.tar.gz" - - cleanup_restore() { - rm -rf "$staging_dir" - if [ "$generated_passphrase" = 'true' ]; then - rm -f "$passphrase_file" - fi - } + restore_passphrase_file="$(make_passphrase_file "$passphrase_arg" read)" + restore_staging_dir="$(mktemp -d "$install_root/backups/.restore-staging.XXXXXX")" + chmod 700 "$restore_staging_dir" + encrypted_tar="$restore_staging_dir/bundle.tar.gz" trap cleanup_restore EXIT log "Decrypting and validating backup before changing the deployment" if ! openssl enc -d -aes-256-cbc -pbkdf2 -iter 200000 -md sha256 \ - -pass "file:$passphrase_file" -in "$backup_file" -out "$encrypted_tar"; then + -pass "file:$restore_passphrase_file" -in "$backup_file" -out "$encrypted_tar"; then die "could not decrypt backup (wrong passphrase or damaged bundle)" fi validate_archive_paths "$encrypted_tar" - tar -xzf "$encrypted_tar" --no-same-owner --no-same-permissions -C "$staging_dir" + tar -xzf "$encrypted_tar" --no-same-owner --no-same-permissions -C "$restore_staging_dir" rm -f "$encrypted_tar" - [ -f "$staging_dir/manifest.json" ] || die "backup manifest is missing" - [ -f "$staging_dir/SHA256SUMS" ] || die "backup checksums are missing" - [ -f "$staging_dir/postgres.sql" ] || die "PostgreSQL dump is missing" - [ -f "$staging_dir/config/.env" ] || die "deployment configuration is missing" - [ -f "$staging_dir/config/docker-compose.prod.yml" ] || die "deployment Compose file is missing" - [ -f "$staging_dir/images.txt" ] || die "image identities are missing" - grep -Eq '"format"[[:space:]]*:[[:space:]]*"roomote-backup"' "$staging_dir/manifest.json" || die "not a Roomote backup bundle" - grep -Eq '"formatVersion"[[:space:]]*:[[:space:]]*1([,[:space:]]|$)' "$staging_dir/manifest.json" || die "unsupported Roomote backup format version" - (trap - EXIT; cd "$staging_dir" && sha256sum -c SHA256SUMS) + [ -f "$restore_staging_dir/manifest.json" ] || die "backup manifest is missing" + [ -f "$restore_staging_dir/SHA256SUMS" ] || die "backup checksums are missing" + [ -f "$restore_staging_dir/postgres.sql" ] || die "PostgreSQL dump is missing" + [ -f "$restore_staging_dir/config/.env" ] || die "deployment configuration is missing" + [ -f "$restore_staging_dir/config/docker-compose.prod.yml" ] || die "deployment Compose file is missing" + [ -f "$restore_staging_dir/images.txt" ] || die "image identities are missing" + grep -Eq '"format"[[:space:]]*:[[:space:]]*"roomote-backup"' "$restore_staging_dir/manifest.json" || die "not a Roomote backup bundle" + grep -Eq '"formatVersion"[[:space:]]*:[[:space:]]*1([,[:space:]]|$)' "$restore_staging_dir/manifest.json" || die "unsupported Roomote backup format version" + (trap - EXIT; cd "$restore_staging_dir" && sha256sum -c SHA256SUMS) if awk ' /"redis"[[:space:]]*:/ { in_redis = 1; next } in_redis && /"included"[[:space:]]*:[[:space:]]*true/ { found = 1; exit } in_redis && /}/ { exit } END { exit !found } - ' "$staging_dir/manifest.json"; then - [ -f "$staging_dir/redis-data.tar" ] || die "Redis is marked as included but its snapshot is missing" - validate_volume_archive "$staging_dir/redis-data.tar" + ' "$restore_staging_dir/manifest.json"; then + [ -f "$restore_staging_dir/redis-data.tar" ] || die "Redis is marked as included but its snapshot is missing" + validate_volume_archive "$restore_staging_dir/redis-data.tar" redis_included='true' fi - if grep -Eq '"mode"[[:space:]]*:[[:space:]]*"local-minio"' "$staging_dir/manifest.json"; then - [ -f "$staging_dir/minio-data.tar" ] || die "local MinIO data is missing" - validate_volume_archive "$staging_dir/minio-data.tar" + if grep -Eq '"mode"[[:space:]]*:[[:space:]]*"local-minio"' "$restore_staging_dir/manifest.json"; then + [ -f "$restore_staging_dir/minio-data.tar" ] || die "local MinIO data is missing" + validate_volume_archive "$restore_staging_dir/minio-data.tar" local_minio_included='true' - elif grep -Eq '"mode"[[:space:]]*:[[:space:]]*"external-bucket"' "$staging_dir/manifest.json"; then + elif grep -Eq '"mode"[[:space:]]*:[[:space:]]*"external-bucket"' "$restore_staging_dir/manifest.json"; then log "This backup references external object storage; ensure its bucket is available before continuing" else die "backup has an unsupported object storage mode" fi log "Restoring recorded container image identities" - restore_image_identities "$staging_dir/images.txt" + restore_image_identities "$restore_staging_dir/images.txt" log "Stopping the existing deployment" stop_task_workers compose down log "Restoring deployment configuration and release metadata" - install -m 600 "$staging_dir/config/.env" "$env_file" - install -m 600 "$staging_dir/config/docker-compose.prod.yml" "$compose_file" - if [ -f "$staging_dir/config/caddy/Caddyfile" ]; then + install -m 600 "$restore_staging_dir/config/.env" "$env_file" + # The bundled .env carries the source host's ROOMOTE_DOCKER_BIN, which may + # not exist here (snap vs apt Docker). Record the binary this restore is + # provably using so later CLI and systemd starts work on this host. + restored_docker_bin="$docker_bin" + if [ "$restored_docker_bin" = 'docker' ]; then + restored_docker_bin="$(command -v docker || printf 'docker')" + fi + set_env_value ROOMOTE_DOCKER_BIN "$restored_docker_bin" + install -m 600 "$restore_staging_dir/config/docker-compose.prod.yml" "$compose_file" + # Put bundled overrides back before the first compose call against the + # restored .env: its COMPOSE_FILE may reference them. Bundles from before + # the override registry simply have no compose-overrides directory. + if [ -d "$restore_staging_dir/config/compose-overrides" ]; then + for entry in "$restore_staging_dir/config/compose-overrides"/*; do + [ -f "$entry" ] || continue + install -m 600 "$entry" "$install_root/${entry##*/}" + done + fi + if [ -f "$restore_staging_dir/config/caddy/Caddyfile" ]; then mkdir -p "$install_root/caddy" - install -m 600 "$staging_dir/config/caddy/Caddyfile" "$install_root/caddy/Caddyfile" + install -m 600 "$restore_staging_dir/config/caddy/Caddyfile" "$install_root/caddy/Caddyfile" fi - if [ -f "$staging_dir/config/deployment.env" ]; then - install -m 600 "$staging_dir/config/deployment.env" "$install_root/deployment.env" + if [ -f "$restore_staging_dir/config/deployment.env" ]; then + install -m 600 "$restore_staging_dir/config/deployment.env" "$install_root/deployment.env" fi compose create redis minio >/dev/null redis_volume="$(container_volume_name redis /data)" if [ "$redis_included" = 'true' ]; then log "Restoring Redis snapshot" - restore_volume "$redis_volume" "$staging_dir" redis-data.tar + restore_volume "$redis_volume" "$restore_staging_dir" redis-data.tar else log "Redis was not included; clearing transient queue and session state" - restore_volume "$redis_volume" "$staging_dir" + restore_volume "$redis_volume" "$restore_staging_dir" fi if [ "$local_minio_included" = 'true' ]; then minio_volume="$(container_volume_name minio /data)" log "Restoring local MinIO artifacts" - restore_volume "$minio_volume" "$staging_dir" minio-data.tar + restore_volume "$minio_volume" "$restore_staging_dir" minio-data.tar fi log "Starting deployment infrastructure" @@ -1018,18 +1314,22 @@ cmd_restore() { log "Restoring PostgreSQL" postgres_command '' 'psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;"' - postgres_command "$staging_dir/postgres.sql" 'psql "$DATABASE_URL" -v ON_ERROR_STOP=1' + postgres_command "$restore_staging_dir/postgres.sql" 'psql "$DATABASE_URL" -v ON_ERROR_STOP=1' log "Starting the restored Roomote release" compose up -d --wait --wait-timeout 600 - rm -rf "$staging_dir" - if [ "$generated_passphrase" = 'true' ]; then - rm -f "$passphrase_file" - fi trap - EXIT + cleanup_restore log "Restore complete" } +# The Docker binary recorded by the installer (snap and manual installs live +# outside systemd's minimal PATH). Resolved here, after read_env_value is +# defined; the ROOMOTE_DOCKER_BIN environment variable wins for tests and +# one-off invocations. +docker_bin="${ROOMOTE_DOCKER_BIN:-$(read_env_value ROOMOTE_DOCKER_BIN)}" +docker_bin="${docker_bin:-docker}" + command="${1:-help}" shift || true @@ -1044,6 +1344,10 @@ case "$command" in restart) compose restart "$@" ;; down) compose down ;; up) compose up -d --wait --wait-timeout 600 ;; + override) cmd_override "$@" ;; + compose) compose "$@" ;; + docker) docker_cmd "$@" ;; + sync-unit) cmd_sync_unit "$@" ;; help | --help | -h) usage ;; *) usage >&2 diff --git a/deploy/host/tests/backup-failed-restart.sh b/deploy/host/tests/backup-failed-restart.sh new file mode 100755 index 000000000..02a0df2d0 --- /dev/null +++ b/deploy/host/tests/backup-failed-restart.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd)" +roomote_cli="$repo_root/deploy/host/roomote" +work_dir="$(mktemp -d)" +install_root="$work_dir/install" +fake_bin="$work_dir/bin" +docker_log="$work_dir/docker.log" +output_log="$work_dir/output.log" +passphrase_file="$work_dir/passphrase" +bundle="$work_dir/backup.roomote" + +cleanup() { + rm -rf "$work_dir" +} +trap cleanup EXIT + +mkdir -p "$install_root/backups" "$install_root/caddy" "$fake_bin" +cat >"$install_root/.env" <<'EOF' +DATABASE_URL=postgres://roomote.example.invalid/roomote +S3_ENDPOINT=https://objects.example.invalid +S3_BUCKET_ARTIFACTS=roomote-artifacts +ROOMOTE_VERSION=backup-test +COMPOSE_FILE=docker-compose.prod.yml:docker-compose.caddy-dns.yml +EOF +printf 'services: {}\n' >"$install_root/docker-compose.prod.yml" +printf 'services: {}\n' >"$install_root/docker-compose.caddy-dns.yml" +printf 'test caddy configuration\n' >"$install_root/caddy/Caddyfile" +printf 'test passphrase\n' >"$passphrase_file" + +# macOS has shasum but not sha256sum; give the CLI what it checks for. +if ! command -v sha256sum >/dev/null 2>&1; then + printf '#!/usr/bin/env bash\nexec shasum -a 256 "$@"\n' >"$fake_bin/sha256sum" + chmod +x "$fake_bin/sha256sum" +fi + +cat >"$fake_bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf '%s\n' "$*" >>"$MOCK_DOCKER_LOG" + +if [ "${1:-}" = 'compose' ]; then + case " $* " in + *' config --images '*) exit 0 ;; + *' up -d --wait --wait-timeout 600 '*) exit 1 ;; + *) exit 0 ;; + esac +fi + +if [ "${1:-}" = 'ps' ]; then + exit 0 +fi + +if [ "${1:-}" = 'run' ]; then + case "$*" in + *pg_dump*) printf '%s\n' '-- database dump' ;; + *'SELECT hash'*) printf '%s\n' 'migration-hash' ;; + esac +fi +EOF +chmod +x "$fake_bin/docker" + +set +e +env PATH="$fake_bin:$PATH" \ + MOCK_DOCKER_LOG="$docker_log" \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" backup --passphrase-file "$passphrase_file" --output "$bundle" >"$output_log" 2>&1 +status=$? +set -e + +[ "$status" -ne 0 ] +grep -q 'Restarting Roomote after interrupted backup' "$output_log" +if grep -q 'unbound variable' "$output_log"; then + printf 'backup cleanup referenced function-local state after shell exit\n' >&2 + exit 1 +fi +# The interrupted-backup restart must include the registered override. +grep -Fq -- "-f $install_root/docker-compose.prod.yml -f $install_root/docker-compose.caddy-dns.yml up -d --wait --wait-timeout 600" "$docker_log" +if compgen -G "$install_root/backups/.backup-staging.*" >/dev/null; then + printf 'backup staging directory was not cleaned up\n' >&2 + exit 1 +fi + +# The bundle written before the failed restart must contain the override. +tar_list="$(openssl enc -d -aes-256-cbc -pbkdf2 -iter 200000 -md sha256 \ + -pass "file:$passphrase_file" -in "$bundle" | tar -tzf -)" +printf '%s\n' "$tar_list" | grep -Fq 'config/compose-overrides/docker-compose.caddy-dns.yml' + +printf 'Failed backup restart retained cleanup state, overrides, and staged them in the bundle.\n' diff --git a/deploy/host/tests/compose-docker-path.sh b/deploy/host/tests/compose-docker-path.sh new file mode 100755 index 000000000..fe895fc72 --- /dev/null +++ b/deploy/host/tests/compose-docker-path.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd)" +roomote_cli="$repo_root/deploy/host/roomote" +work_dir="$(mktemp -d)" +install_root="$work_dir/install" +custom_bin="$work_dir/custom-bin/docker-custom" +docker_log="$work_dir/docker.log" + +cleanup() { + rm -rf "$work_dir" +} +trap cleanup EXIT + +mkdir -p "$install_root" "$(dirname -- "$custom_bin")" +printf 'ROOMOTE_VERSION=test\n' >"$install_root/.env" +printf 'services: {}\n' >"$install_root/docker-compose.prod.yml" + +cat >"$custom_bin" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >"$MOCK_DOCKER_LOG" +EOF +chmod +x "$custom_bin" + +# PATH deliberately has no `docker`: only the recorded binary may be used. + +# 1. ROOMOTE_DOCKER_BIN from the environment (systemd-style override). +env PATH='/usr/bin:/bin' \ + MOCK_DOCKER_LOG="$docker_log" \ + ROOMOTE_DOCKER_BIN="$custom_bin" \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" status +grep -Fq -- "compose --env-file $install_root/.env -f $install_root/docker-compose.prod.yml ps" "$docker_log" + +# 2. ROOMOTE_DOCKER_BIN persisted in .env by the installer. +printf 'ROOMOTE_DOCKER_BIN=%s\n' "$custom_bin" >>"$install_root/.env" +: >"$docker_log" +env PATH='/usr/bin:/bin' \ + MOCK_DOCKER_LOG="$docker_log" \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" status +grep -Fq -- "compose --env-file $install_root/.env -f $install_root/docker-compose.prod.yml ps" "$docker_log" + +# 3. Non-compose Docker invocations honor it too (docker_cmd everywhere): +# `restart` goes through compose; exercise a raw docker path via backup's +# worker scan by calling status is not enough, so probe `logs` and a direct +# ps-based command. The compose passthrough covers arbitrary invocations. +: >"$docker_log" +env PATH='/usr/bin:/bin' \ + MOCK_DOCKER_LOG="$docker_log" \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" compose ps -aq redis +grep -Fq -- "compose --env-file $install_root/.env -f $install_root/docker-compose.prod.yml ps -aq redis" "$docker_log" + +# The `roomote docker` passthrough (used by managed deploys for the worker +# image pull) resolves the recorded binary too. +: >"$docker_log" +env PATH='/usr/bin:/bin' \ + MOCK_DOCKER_LOG="$docker_log" \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" docker pull example.invalid/roomote-worker:test +grep -Fq -- "pull example.invalid/roomote-worker:test" "$docker_log" + +# 4. sync-unit records the resolved Docker path when .env lacks one, so a +# systemd boot (minimal PATH) can start the stack on snap/manual installs. +# Managed deploys call sync-unit without the installer, so the guarantee +# must hold from the CLI alone. +systemd_dir="$work_dir/systemd" +docker_dir="$work_dir/path-docker" +mkdir -p "$systemd_dir" "$docker_dir" +printf '#!/usr/bin/env bash\nexit 0\n' >"$docker_dir/docker" +chmod +x "$docker_dir/docker" +grep -v '^ROOMOTE_DOCKER_BIN=' "$install_root/.env" >"$install_root/.env.tmp" +mv "$install_root/.env.tmp" "$install_root/.env" +env PATH="$docker_dir:/usr/bin:/bin" \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_SYSTEMD_DIR="$systemd_dir" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" sync-unit >/dev/null +grep -Fq "ROOMOTE_DOCKER_BIN=$docker_dir/docker" "$install_root/.env" +grep -Fq 'ExecStart=/usr/local/bin/roomote up' "$systemd_dir/roomote-compose.service" + +printf 'Host CLI used the recorded Docker binary for every invocation.\n' diff --git a/deploy/host/tests/compose-overrides.sh b/deploy/host/tests/compose-overrides.sh new file mode 100755 index 000000000..391cd1c7f --- /dev/null +++ b/deploy/host/tests/compose-overrides.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd)" +roomote_cli="$repo_root/deploy/host/roomote" +work_dir="$(mktemp -d)" +install_root="$work_dir/install" +fake_bin="$work_dir/bin" +docker_log="$work_dir/docker.log" +output_log="$work_dir/output.log" + +cleanup() { + rm -rf "$work_dir" +} +trap cleanup EXIT + +mkdir -p "$install_root" "$fake_bin" +cat >"$install_root/.env" <<'EOF' +ROOMOTE_VERSION=test +COMPOSE_FILE=docker-compose.prod.yml:docker-compose.caddy-dns.yml +EOF +printf 'services: {}\n' >"$install_root/docker-compose.prod.yml" +printf 'services: {}\n' >"$install_root/docker-compose.caddy-dns.yml" +printf 'services: {}\n' >"$install_root/docker-compose.extra.yml" +# Present on disk but not registered: must never be picked up implicitly. +printf 'services: {}\n' >"$install_root/docker-compose.stray.yml" + +cat >"$fake_bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"$MOCK_DOCKER_LOG" +EOF +chmod +x "$fake_bin/docker" + +run_cli() { + env PATH="$fake_bin:$PATH" \ + MOCK_DOCKER_LOG="$docker_log" \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" "$@" +} + +# Registered overrides are appended in order; unregistered files are ignored. +run_cli status +grep -Fq -- "compose --env-file $install_root/.env -f $install_root/docker-compose.prod.yml -f $install_root/docker-compose.caddy-dns.yml ps" "$docker_log" +if grep -q 'stray' "$docker_log"; then + printf 'unregistered compose file was passed to docker compose\n' >&2 + exit 1 +fi + +# override list prints the merge order. +run_cli override list >"$output_log" +grep -Fq 'docker-compose.prod.yml (managed base file)' "$output_log" +grep -Fq 'docker-compose.caddy-dns.yml' "$output_log" + +# override add validates the merged config, then records the entry. +run_cli override add docker-compose.extra.yml >"$output_log" +grep -Fq 'COMPOSE_FILE=docker-compose.prod.yml:docker-compose.caddy-dns.yml:docker-compose.extra.yml' "$install_root/.env" +grep -Fq -- "-f $install_root/docker-compose.extra.yml config" "$docker_log" + +# Adding a file that does not exist fails without touching the registry. +if run_cli override add docker-compose.missing.yml >"$output_log" 2>&1; then + printf 'override add accepted a missing file\n' >&2 + exit 1 +fi +grep -Fq 'COMPOSE_FILE=docker-compose.prod.yml:docker-compose.caddy-dns.yml:docker-compose.extra.yml' "$install_root/.env" + +# override remove drops the entry but keeps the file. +run_cli override remove docker-compose.extra.yml >"$output_log" +grep -Fq 'COMPOSE_FILE=docker-compose.prod.yml:docker-compose.caddy-dns.yml' "$install_root/.env" +[ -f "$install_root/docker-compose.extra.yml" ] + +# The base file cannot be removed. +if run_cli override remove docker-compose.prod.yml >"$output_log" 2>&1; then + printf 'override remove accepted the managed base file\n' >&2 + exit 1 +fi + +# A registered-but-missing file fails loudly with the fix spelled out. +rm "$install_root/docker-compose.caddy-dns.yml" +if run_cli status >"$output_log" 2>&1; then + printf 'status succeeded despite a missing registered override\n' >&2 + exit 1 +fi +grep -Fq "roomote override remove docker-compose.caddy-dns.yml" "$output_log" + +# COMPOSE_FILE must keep the managed base file first. +printf 'services: {}\n' >"$install_root/docker-compose.caddy-dns.yml" +sed -i.bak 's/^COMPOSE_FILE=.*/COMPOSE_FILE=docker-compose.caddy-dns.yml:docker-compose.prod.yml/' "$install_root/.env" +if run_cli status >"$output_log" 2>&1; then + printf 'status accepted a COMPOSE_FILE without the base file first\n' >&2 + exit 1 +fi +grep -Fq 'must list the managed base file' "$output_log" + +printf 'Compose override registry behaves as documented.\n' diff --git a/deploy/host/tests/restore-failed-cleanup.sh b/deploy/host/tests/restore-failed-cleanup.sh new file mode 100755 index 000000000..62d0ecc52 --- /dev/null +++ b/deploy/host/tests/restore-failed-cleanup.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd)" +roomote_cli="$repo_root/deploy/host/roomote" +work_dir="$(mktemp -d)" +install_root="$work_dir/install" +fake_bin="$work_dir/bin" +docker_log="$work_dir/docker.log" +output_log="$work_dir/output.log" +tmp_dir="$work_dir/tmp" +bundle="$work_dir/backup.roomote" + +cleanup() { + rm -rf "$work_dir" +} +trap cleanup EXIT + +mkdir -p "$install_root/backups" "$install_root/caddy" "$fake_bin" "$tmp_dir" +cat >"$install_root/.env" <<'EOF' +DATABASE_URL=postgres://roomote.example.invalid/roomote +S3_ENDPOINT=https://objects.example.invalid +S3_BUCKET_ARTIFACTS=roomote-artifacts +ROOMOTE_VERSION=restore-test +EOF +printf 'services: {}\n' >"$install_root/docker-compose.prod.yml" +printf 'test caddy configuration\n' >"$install_root/caddy/Caddyfile" + +if ! command -v sha256sum >/dev/null 2>&1; then + printf '#!/usr/bin/env bash\nexec shasum -a 256 "$@"\n' >"$fake_bin/sha256sum" + chmod +x "$fake_bin/sha256sum" +fi + +cat >"$fake_bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf '%s\n' "$*" >>"$MOCK_DOCKER_LOG" + +if [ "${1:-}" = 'compose' ]; then + if [ "${MOCK_FAIL_DOWN:-}" = 'true' ]; then + case " $* " in + *' down '*) exit 1 ;; + esac + fi + case " $* " in + *' ps -aq '*) printf 'mock-container\n' ;; + esac + exit 0 +fi + +if [ "${1:-}" = 'inspect' ]; then + printf 'mock-volume\n' + exit 0 +fi + +if [ "${1:-}" = 'ps' ]; then + exit 0 +fi + +if [ "${1:-}" = 'run' ]; then + case "$*" in + *pg_dump*) printf '%s\n' '-- database dump' ;; + *'SELECT hash'*) printf '%s\n' 'migration-hash' ;; + esac +fi +EOF +chmod +x "$fake_bin/docker" + +# The bundle should carry a source-host Docker path that does not exist here, +# to prove restore re-records the recovery host's binary. +printf 'ROOMOTE_DOCKER_BIN=/nonexistent/source-docker\n' >>"$install_root/.env" + +# Create a valid bundle first (everything mocked to succeed). The environment +# override wins over the .env value, like the systemd unit's environment does. +env PATH="$fake_bin:$PATH" \ + MOCK_DOCKER_LOG="$docker_log" \ + TMPDIR="$tmp_dir" \ + ROOMOTE_BACKUP_PASSPHRASE='restore-cleanup-passphrase' \ + ROOMOTE_DOCKER_BIN="$fake_bin/docker" \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" backup --output "$bundle" >"$output_log" 2>&1 + +# A successful run must leave no passphrase temp files behind either. +if [ -n "$(ls -A "$tmp_dir")" ]; then + printf 'backup left temp files behind: %s\n' "$(ls -A "$tmp_dir")" >&2 + exit 1 +fi + +# Now fail the restore partway (compose down) and check the EXIT trap. +set +e +env PATH="$fake_bin:$PATH" \ + MOCK_DOCKER_LOG="$docker_log" \ + MOCK_FAIL_DOWN=true \ + TMPDIR="$tmp_dir" \ + ROOMOTE_BACKUP_PASSPHRASE='restore-cleanup-passphrase' \ + ROOMOTE_DOCKER_BIN="$fake_bin/docker" \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" restore "$bundle" --yes >"$output_log" 2>&1 +status=$? +set -e + +[ "$status" -ne 0 ] +if grep -q 'unbound variable' "$output_log"; then + printf 'restore cleanup referenced function-local state after shell exit\n' >&2 + exit 1 +fi +if compgen -G "$install_root/backups/.restore-staging.*" >/dev/null; then + printf 'restore staging directory (with decrypted postgres.sql) was not cleaned up\n' >&2 + exit 1 +fi +if [ -n "$(ls -A "$tmp_dir")" ]; then + printf 'restore left the passphrase temp file behind: %s\n' "$(ls -A "$tmp_dir")" >&2 + exit 1 +fi + +# A successful restore must re-record this host's Docker binary instead of +# keeping the source host's absolute path from the bundled .env. +env PATH="$fake_bin:$PATH" \ + MOCK_DOCKER_LOG="$docker_log" \ + TMPDIR="$tmp_dir" \ + ROOMOTE_BACKUP_PASSPHRASE='restore-cleanup-passphrase' \ + ROOMOTE_DOCKER_BIN="$fake_bin/docker" \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" restore "$bundle" --yes >"$output_log" 2>&1 +grep -q 'Restore complete' "$output_log" +grep -Fq "ROOMOTE_DOCKER_BIN=$fake_bin/docker" "$install_root/.env" +if grep -q '/nonexistent/source-docker' "$install_root/.env"; then + printf 'restore kept the source host Docker path in .env\n' >&2 + exit 1 +fi + +printf 'Failed restore cleaned up; successful restore re-recorded the Docker path.\n' diff --git a/deploy/host/tests/upgrade-failed-pull.sh b/deploy/host/tests/upgrade-failed-pull.sh index acb8ccab9..12a3f74de 100755 --- a/deploy/host/tests/upgrade-failed-pull.sh +++ b/deploy/host/tests/upgrade-failed-pull.sh @@ -6,6 +6,8 @@ roomote_cli="$repo_root/deploy/host/roomote" work_dir="$(mktemp -d)" install_root="$work_dir/install" fake_bin="$work_dir/bin" +cli_path="$work_dir/cli/roomote" +systemd_dir="$work_dir/systemd" docker_log="$work_dir/docker.log" output_log="$work_dir/output.log" @@ -14,7 +16,8 @@ cleanup() { } trap cleanup EXIT -mkdir -p "$install_root/backups" "$install_root/caddy" "$fake_bin" +mkdir -p "$install_root/backups" "$install_root/caddy" "$fake_bin" \ + "$(dirname -- "$cli_path")" "$systemd_dir" cat >"$install_root/.env" <<'EOF' ROOMOTE_REPO=RooCodeInc/Roomote ROOMOTE_VERSION=v1.0.0 @@ -25,12 +28,19 @@ IMAGE_NAMESPACE=roocodeinc DOCKER_WORKER_IMAGE=ghcr.io/roocodeinc/roomote-worker:v1.0.0 MODAL_BASE_IMAGE_REF=ghcr.io/roocodeinc/roomote-worker:v1.0.0 R_DISCORD_GATEWAY_SECRET=test-secret +COMPOSE_FILE=docker-compose.prod.yml:docker-compose.caddy-dns.yml EOF printf 'original compose\n' >"$install_root/docker-compose.prod.yml" +printf 'custom caddy override\n' >"$install_root/docker-compose.caddy-dns.yml" printf 'original caddy\n' >"$install_root/caddy/Caddyfile" +printf 'previous cli\n' >"$cli_path" +chmod +x "$cli_path" +printf 'previous unit\n' >"$systemd_dir/roomote-compose.service" cp "$install_root/.env" "$work_dir/original.env" cp "$install_root/docker-compose.prod.yml" "$work_dir/original-compose.yml" cp "$install_root/caddy/Caddyfile" "$work_dir/original-Caddyfile" +cp "$cli_path" "$work_dir/original-cli" +cp "$systemd_dir/roomote-compose.service" "$work_dir/original-unit" cat >"$fake_bin/curl" <<'EOF' #!/usr/bin/env bash @@ -55,6 +65,7 @@ done case "$url" in */docker-compose.prod.yml) printf 'refreshed compose\n' >"$output" ;; */Caddyfile) printf 'refreshed caddy\n' >"$output" ;; + */deploy/host/roomote) cat "$MOCK_REAL_CLI" >"$output" ;; *) exit 1 ;; esac EOF @@ -64,7 +75,7 @@ cat >"$fake_bin/docker" <<'EOF' set -euo pipefail printf '%s\n' "$*" >>"$MOCK_DOCKER_LOG" -if [ "${1:-}" = 'pull' ]; then +if [ "${1:-}" = 'compose' ] && [ "${*: -1}" = 'pull' ]; then exit 1 fi EOF @@ -74,8 +85,11 @@ chmod +x "$fake_bin/curl" "$fake_bin/docker" set +e env PATH="$fake_bin:$PATH" \ MOCK_DOCKER_LOG="$docker_log" \ + MOCK_REAL_CLI="$roomote_cli" \ ROOMOTE_FETCH_BASE='https://example.invalid' \ ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_CLI_PATH="$cli_path" \ + ROOMOTE_SYSTEMD_DIR="$systemd_dir" \ ROOMOTE_TEST_MODE=true \ "$roomote_cli" upgrade missing-tag --skip-backup >"$output_log" 2>&1 status=$? @@ -85,11 +99,17 @@ set -e cmp "$work_dir/original.env" "$install_root/.env" cmp "$work_dir/original-compose.yml" "$install_root/docker-compose.prod.yml" cmp "$work_dir/original-Caddyfile" "$install_root/caddy/Caddyfile" +# A failed upgrade must also put the previous CLI and systemd unit back. +cmp "$work_dir/original-cli" "$cli_path" +cmp "$work_dir/original-unit" "$systemd_dir/roomote-compose.service" grep -q '^compose .* start controller$' "$docker_log" +# Every compose step of the attempted upgrade carried the operator override. +grep -Fq -- "-f $install_root/docker-compose.prod.yml -f $install_root/docker-compose.caddy-dns.yml config" "$docker_log" +grep -Fq -- "-f $install_root/docker-compose.prod.yml -f $install_root/docker-compose.caddy-dns.yml pull" "$docker_log" grep -q 'Upgrade failed; restoring the previous deployment configuration' "$output_log" if compgen -G "$install_root/backups/.upgrade-staging.*" >/dev/null; then printf 'upgrade staging directory was not cleaned up\n' >&2 exit 1 fi -printf 'Failed image pull preserved the deployed release metadata.\n' +printf 'Failed image pull preserved the deployed release metadata, CLI, and unit.\n' diff --git a/deploy/host/tests/upgrade-refresh.sh b/deploy/host/tests/upgrade-refresh.sh new file mode 100755 index 000000000..f2eb7d8b3 --- /dev/null +++ b/deploy/host/tests/upgrade-refresh.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd)" +roomote_cli="$repo_root/deploy/host/roomote" +work_dir="$(mktemp -d)" +install_root="$work_dir/install" +fake_bin="$work_dir/bin" +cli_path="$work_dir/cli/roomote" +systemd_dir="$work_dir/systemd" +docker_log="$work_dir/docker.log" +output_log="$work_dir/output.log" + +cleanup() { + rm -rf "$work_dir" +} +trap cleanup EXIT + +mkdir -p "$install_root/backups" "$install_root/caddy" "$fake_bin" \ + "$(dirname -- "$cli_path")" "$systemd_dir" +cat >"$install_root/.env" <<'EOF' +ROOMOTE_REPO=RooCodeInc/Roomote +ROOMOTE_VERSION=v1.0.0 +ROOMOTE_APP_DOMAIN=roomote.example.com +IMAGE_REGISTRY=ghcr.io +IMAGE_NAMESPACE=roocodeinc +DOCKER_WORKER_IMAGE=ghcr.io/roocodeinc/roomote-worker:v1.0.0 +MODAL_BASE_IMAGE_REF=ghcr.io/roocodeinc/roomote-worker:v1.0.0 +R_DISCORD_GATEWAY_SECRET=test-secret +COMPOSE_FILE=docker-compose.prod.yml:docker-compose.caddy-dns.yml +EOF +printf 'original compose\n' >"$install_root/docker-compose.prod.yml" +printf 'custom caddy override\n' >"$install_root/docker-compose.caddy-dns.yml" +printf 'original caddy\n' >"$install_root/caddy/Caddyfile" +cp "$install_root/docker-compose.caddy-dns.yml" "$work_dir/original-override.yml" +printf '#!/usr/bin/env bash\nexit 0\n' >"$cli_path" +chmod +x "$cli_path" +printf 'old unit with direct docker compose ExecStart\n' >"$systemd_dir/roomote-compose.service" + +cat >"$fake_bin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +output='' +url='' +while [ "$#" -gt 0 ]; do + case "$1" in + -o) + output="$2" + shift 2 + ;; + http*) + url="$1" + shift + ;; + *) shift ;; + esac +done + +case "$url" in + */docker-compose.prod.yml) printf 'refreshed compose\n' >"$output" ;; + */Caddyfile) printf 'refreshed caddy\n' >"$output" ;; + */deploy/host/roomote) cat "$MOCK_REAL_CLI" >"$output" ;; + *) exit 1 ;; +esac +EOF + +cat >"$fake_bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"$MOCK_DOCKER_LOG" +EOF + +chmod +x "$fake_bin/curl" "$fake_bin/docker" + +env PATH="$fake_bin:$PATH" \ + MOCK_DOCKER_LOG="$docker_log" \ + MOCK_REAL_CLI="$roomote_cli" \ + ROOMOTE_FETCH_BASE='https://example.invalid' \ + ROOMOTE_INSTALL_ROOT="$install_root" \ + ROOMOTE_CLI_PATH="$cli_path" \ + ROOMOTE_SYSTEMD_DIR="$systemd_dir" \ + ROOMOTE_TEST_MODE=true \ + "$roomote_cli" upgrade v1.1.0 --skip-backup >"$output_log" 2>&1 + +# The upgrade must deliver the new CLI and rewrite the systemd unit. +cmp "$roomote_cli" "$cli_path" +grep -Fq "ExecStart=$cli_path up" "$systemd_dir/roomote-compose.service" +grep -Fq "ExecStop=$cli_path down" "$systemd_dir/roomote-compose.service" +grep -Fq "ROOMOTE_INSTALL_ROOT=$install_root" "$systemd_dir/roomote-compose.service" +if grep -q 'direct docker compose ExecStart' "$systemd_dir/roomote-compose.service"; then + printf 'old systemd unit content survived the upgrade\n' >&2 + exit 1 +fi + +# Operator override survives and is part of the final start. +cmp "$work_dir/original-override.yml" "$install_root/docker-compose.caddy-dns.yml" +grep -Fq -- "-f $install_root/docker-compose.prod.yml -f $install_root/docker-compose.caddy-dns.yml up -d --wait --wait-timeout 600" "$docker_log" +grep -Fq -- "-f $install_root/docker-compose.prod.yml -f $install_root/docker-compose.caddy-dns.yml pull" "$docker_log" + +# Migration seeding for pre-registry installs (values already present are kept). +grep -Fq 'COMPOSE_FILE=docker-compose.prod.yml:docker-compose.caddy-dns.yml' "$install_root/.env" +grep -Eq '^ROOMOTE_DOCKER_BIN=' "$install_root/.env" +grep -Fq 'ROOMOTE_VERSION=v1.1.0' "$install_root/.env" + +printf 'Upgrade refreshed the host CLI and systemd unit and preserved overrides.\n' diff --git a/deploy/install.sh b/deploy/install.sh index 74f1923a4..8a8e384c9 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -611,47 +611,41 @@ set_env_value DOCKER_WORKER_NETWORK 'roomote_worker' # baked RELEASE_VERSION differ. set_env_value DOCKER_WORKER_RELEASE_PATH '/roomote/releases/worker-current.tar.gz' +# COMPOSE_FILE records the deployment's Compose file list (base file plus any +# operator overrides registered with `roomote override add`). Preserve an +# existing list on reruns -- overrides are operator state, like TLS mode. +if ! env_has_value COMPOSE_FILE; then + set_env_value COMPOSE_FILE docker-compose.prod.yml +fi + +# The host CLI reads the Docker binary path from .env so every operation -- +# including systemd starts with their minimal PATH -- works with nonstandard +# Docker installs (snap, manual binary). Refreshed on every rerun. +set_env_value ROOMOTE_DOCKER_BIN "$(command -v docker)" + chown root:root "$env_file" chmod 600 "$env_file" # --- systemd unit ------------------------------------------------------------ -# systemd ExecStart needs an absolute path, and a pre-existing Docker install -# may live outside /usr/bin (snap, manual binary). -docker_bin="$(command -v docker)" - -cat >/etc/systemd/system/roomote-compose.service </dev/null 2>&1 +# The unit is owned by the host CLI (sync-unit) so boots and systemd restarts +# use the same override-aware Compose invocation as every other operation, and +# so `roomote upgrade` can refresh the unit on existing deployments. +/usr/local/bin/roomote sync-unit # --- Start the stack --------------------------------------------------------- +# Validation, pull, and start all go through the host CLI so they cover the +# full Compose file list (base file plus any operator overrides on a rerun). cd "$install_root" log "Validating the Compose configuration" -docker compose --env-file .env -f docker-compose.prod.yml config >/dev/null +/usr/local/bin/roomote compose config >/dev/null log "Pulling application images (this can take a few minutes)" -docker compose --env-file .env -f docker-compose.prod.yml pull +/usr/local/bin/roomote compose pull log "Starting Roomote" -docker compose --env-file .env -f docker-compose.prod.yml up -d --wait --wait-timeout 600 +/usr/local/bin/roomote up # The worker image is only needed once the first task runs, so it downloads # in the background while the operator finishes setup in the browser. diff --git a/deploy/scripts/deploy.sh b/deploy/scripts/deploy.sh index eb7c73ee9..3730824aa 100755 --- a/deploy/scripts/deploy.sh +++ b/deploy/scripts/deploy.sh @@ -342,25 +342,51 @@ printf 'Copying Compose, Caddy, and env files to %s\n' "$target" ssh "${ssh_args[@]}" "$target" 'install -d -m 0700 /opt/roomote /opt/roomote/caddy /opt/roomote/backups' scp "${scp_args[@]}" "$deploy_root/compose/docker-compose.prod.yml" "$target:/opt/roomote/docker-compose.prod.yml" scp "${scp_args[@]}" "$deploy_root/caddy/Caddyfile" "$target:/opt/roomote/caddy/Caddyfile" +# The host CLI owns the Compose invocation (base file plus operator overrides +# from COMPOSE_FILE in .env) and the systemd unit; ship it with every deploy +# so this path can never diverge from `roomote up`. +scp "${scp_args[@]}" "$deploy_root/host/roomote" "$target:/usr/local/bin/roomote" +ssh "${ssh_args[@]}" "$target" 'chmod 0755 /usr/local/bin/roomote' scp "${scp_args[@]}" "$tmp_env" "$target:/tmp/roomote.env" -ssh "${ssh_args[@]}" "$target" 'mv /tmp/roomote.env /opt/roomote/.env && chown root:root /opt/roomote/.env && chmod 600 /opt/roomote/.env' +ssh "${ssh_args[@]}" "$target" bash -s <<'REMOTE' +set -euo pipefail +# The override registry (COMPOSE_FILE) and the recorded Docker path are +# host-owned state created on the host itself (`roomote override add`, +# sync-unit); a redeploy from the operator's dotenv must not drop them +# unless that dotenv sets them explicitly. +for key in COMPOSE_FILE ROOMOTE_DOCKER_BIN; do + if [ -f /opt/roomote/.env ] && + ! grep -Eq "^[[:space:]]*(export[[:space:]]+)?$key=." /tmp/roomote.env; then + value="$(awk -v key="$key" ' + BEGIN { pattern = "^[[:space:]]*(export[[:space:]]+)?" key "=" } + $0 ~ pattern { + sub(/^[^=]*=/, "") + print + exit + } + ' /opt/roomote/.env)" + [ -z "$value" ] || printf '%s=%s\n' "$key" "$value" >>/tmp/roomote.env + fi +done +mv /tmp/roomote.env /opt/roomote/.env +chown root:root /opt/roomote/.env +chmod 600 /opt/roomote/.env +REMOTE printf 'Pulling images and starting Roomote %s\n' "$roomote_version" ssh "${ssh_args[@]}" "$target" "ROOMOTE_WORKER_IMAGE=$(shell_quote "$worker_image") bash -s" <<'REMOTE' set -euo pipefail : "${ROOMOTE_WORKER_IMAGE:?ROOMOTE_WORKER_IMAGE is required}" cd /opt/roomote -if [ -f /etc/systemd/system/roomote-compose.service ]; then - sed -i '/^EnvironmentFile=-\/opt\/roomote\/deployment.env$/d' /etc/systemd/system/roomote-compose.service - systemctl daemon-reload -fi -docker compose --env-file .env -f docker-compose.prod.yml config >/dev/null +# sync-unit rewrites the whole unit (CLI-owned, override-aware ExecStart), +# which also drops the obsolete EnvironmentFile line older units carried. +/usr/local/bin/roomote sync-unit +/usr/local/bin/roomote compose config >/dev/null echo "Stopping controller before image pull so new tasks remain queued during deploy" -docker compose --env-file .env -f docker-compose.prod.yml stop controller || true -docker pull "$ROOMOTE_WORKER_IMAGE" -docker compose --env-file .env -f docker-compose.prod.yml pull -docker compose --env-file .env -f docker-compose.prod.yml up -d --wait --wait-timeout 600 -systemctl enable roomote-compose.service +/usr/local/bin/roomote compose stop controller || true +/usr/local/bin/roomote docker pull "$ROOMOTE_WORKER_IMAGE" +/usr/local/bin/roomote compose pull +/usr/local/bin/roomote up REMOTE printf 'Pruning old Roomote images on %s; keeping %s release tag(s)\n' "$target" "$image_retention_releases" diff --git a/deploy/scripts/upgrade.sh b/deploy/scripts/upgrade.sh index 63f7a1936..b33e30404 100755 --- a/deploy/scripts/upgrade.sh +++ b/deploy/scripts/upgrade.sh @@ -86,23 +86,27 @@ target="$ssh_user@$host" configure_ssh_args "$ssh_private_key" printf 'Upgrading %s on %s to %s\n' "$customer" "$target" "$version" -printf 'Copying updated Compose and Caddy files to %s\n' "$target" +printf 'Copying updated Compose, Caddy, and host CLI files to %s\n' "$target" ssh "${ssh_args[@]}" "$target" 'install -d -m 0700 /opt/roomote /opt/roomote/caddy' scp "${scp_args[@]}" "$deploy_root/compose/docker-compose.prod.yml" "$target:/opt/roomote/docker-compose.prod.yml" scp "${scp_args[@]}" "$deploy_root/caddy/Caddyfile" "$target:/opt/roomote/caddy/Caddyfile" +# The host CLI owns the Compose invocation (base file plus operator overrides +# from COMPOSE_FILE in .env) and the systemd unit; refresh it with every +# managed upgrade so this path can never diverge from `roomote upgrade`. +scp "${scp_args[@]}" "$deploy_root/host/roomote" "$target:/usr/local/bin/roomote" +ssh "${ssh_args[@]}" "$target" 'chmod 0755 /usr/local/bin/roomote' ssh "${ssh_args[@]}" "$target" \ "ROOMOTE_VERSION=$(shell_quote "$version") ROOMOTE_IMAGE_REGISTRY_ARG=$(shell_quote "$image_registry") ROOMOTE_IMAGE_NAMESPACE_ARG=$(shell_quote "$image_namespace") bash -s" <<'REMOTE' set -euo pipefail cd /opt/roomote -if [ -f /etc/systemd/system/roomote-compose.service ]; then - sed -i '/^EnvironmentFile=-\/opt\/roomote\/deployment.env$/d' /etc/systemd/system/roomote-compose.service - systemctl daemon-reload -fi -docker compose --env-file .env -f docker-compose.prod.yml config >/dev/null +# sync-unit rewrites the whole unit (CLI-owned, override-aware ExecStart), +# which also drops the obsolete EnvironmentFile line older units carried. +/usr/local/bin/roomote sync-unit +/usr/local/bin/roomote compose config >/dev/null echo "Stopping controller before deployment metadata changes so new tasks remain queued during deploy" -docker compose --env-file .env -f docker-compose.prod.yml stop controller || true +/usr/local/bin/roomote compose stop controller || true read_env_value() { local key="$1" @@ -258,21 +262,20 @@ if [ -z "$(read_env_value R_DISCORD_GATEWAY_SECRET | tr -d '[:space:]')" ]; then echo "Generated R_DISCORD_GATEWAY_SECRET for Discord gateway↔API auth" fi -docker compose --env-file .env -f docker-compose.prod.yml config >/dev/null -docker pull "$worker_image" -docker compose --env-file .env -f docker-compose.prod.yml pull +/usr/local/bin/roomote compose config >/dev/null +/usr/local/bin/roomote docker pull "$worker_image" +/usr/local/bin/roomote compose pull # Migrations run before any running service is replaced. Drizzle applies all # pending migrations in a single transaction, so a failure here rolls the # schema back while the previous release keeps serving. echo "Applying database migrations before replacing running services" -if ! docker compose --env-file .env -f docker-compose.prod.yml run --rm db-migrate; then +if ! /usr/local/bin/roomote compose run --rm db-migrate; then echo "Database migrations failed and were rolled back; the previous release keeps serving." >&2 echo "Restarting the previous controller. Re-run roomote-deploy upgrade with the previous tag to restore deployment metadata, or fix the migration and retry." >&2 - docker compose --env-file .env -f docker-compose.prod.yml start controller || true + /usr/local/bin/roomote compose start controller || true exit 1 fi -docker compose --env-file .env -f docker-compose.prod.yml up -d --wait --wait-timeout 600 -systemctl enable roomote-compose.service +/usr/local/bin/roomote up REMOTE printf 'Pruning old Roomote images on %s; keeping %s release tag(s)\n' "$target" "$image_retention_releases"