[Fix] Preserve operator Compose overrides via an explicit registry owned by the host CLI

.github/workflows/CI.yml

@@ -197,6 +197,16 @@ jobs:
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Verify a failed image pull preserves deployment metadata
         run: bash deploy/host/tests/upgrade-failed-pull.sh
+      - name: Verify the Compose override registry
+        run: bash deploy/host/tests/compose-overrides.sh
+      - name: Verify nonstandard Docker binary support
+        run: bash deploy/host/tests/compose-docker-path.sh
+      - name: Verify failed backup restart cleanup
+        run: bash deploy/host/tests/backup-failed-restart.sh
+      - name: Verify failed restore cleanup
+        run: bash deploy/host/tests/restore-failed-cleanup.sh
+      - name: Verify upgrade refreshes the CLI and systemd unit
+        run: bash deploy/host/tests/upgrade-refresh.sh
       - name: Restore an encrypted bundle onto empty volumes
         run: bash deploy/host/tests/backup-restore.integration.sh
 

SELF_HOSTING.md

@@ -147,6 +147,31 @@ mode only when you know public DNS is not ready yet. The selected mode is
 retained in `/opt/roomote/.env`, so installer reruns and `roomote upgrade`
 preserve it.
 
+### Compose overrides
+
+To customize the stack beyond what `.env` covers (for example a Caddy image
+built with a DNS-provider plugin), put your changes in a Compose override file
+next to the managed base file and register it:
+
+```sh
+sudo roomote override add docker-compose.caddy-dns.yml
+```
+
+The file must already exist in `/opt/roomote` as a plain `.yml`/`.yaml` file
+name. Registered overrides are recorded in `COMPOSE_FILE` in
+`/opt/roomote/.env` and layered after `docker-compose.prod.yml` in the listed
+order, for every `roomote` command, systemd start, and boot. They are included
+in backup bundles, restored by `roomote restore`, and preserved across
+`roomote upgrade` (which refreshes only the managed base file and Caddyfile).
+`roomote override list` shows the merge order and `roomote override remove`
+unregisters a file without deleting it.
+
+Never edit `docker-compose.prod.yml` or the managed Caddyfile directly: both
+are replaced on every upgrade. One caveat for overrides that swap in custom
+images: backup bundles record image identities for the managed stack only, so
+a locally built override image must be rebuilt or re-pulled by you after a
+fresh-host restore.
+
 ### Cloudflare Tunnel
 
 Cloudflare Tunnel works with the supported `internal` TLS mode: Cloudflare

apps/docs/self-hosting.mdx

@@ -91,8 +91,28 @@ roomote upgrade   # pull and roll out newer images
 roomote rollback  # return to the release before the last upgrade
 roomote backup    # create an encrypted deployment recovery bundle
 roomote logs      # tail service logs
+roomote override  # register operator Compose override files
 ```
 
+### Customizing the stack with Compose overrides
+
+`docker-compose.prod.yml` and the managed Caddyfile are replaced on every
+upgrade, so never edit them directly. Put customizations (for example a Caddy
+image built with a DNS-provider plugin) in an override file in `/opt/roomote`
+and register it:
+
+```sh
+sudo roomote override add docker-compose.caddy-dns.yml
+```
+
+Registered overrides are layered after the base file, in registration order,
+for every `roomote` command, systemd restart, and boot. They ride along in
+backup bundles, come back with `roomote restore`, and survive `roomote
+upgrade`. `roomote override list` shows the merge order; `roomote override
+remove` unregisters a file without deleting it. Images referenced only by
+override files are not covered by backup image identities, so keep locally
+built override images rebuildable on a replacement host.
+
 `roomote backup` prompts for a passphrase and writes a versioned `.roomote`
 bundle under `/opt/roomote/backups`. The bundle contains PostgreSQL, the
 deployment configuration and encryption/signing keys, local MinIO artifacts,
@@ -136,7 +156,9 @@ empty PostgreSQL/MinIO/Redis volumes, and starts the recorded Roomote release.
   that also rewinds data.
 
 Use `roomote upgrade` instead of updating application image references alone.
-The command refreshes the release's Compose and Caddy configuration together;
+The command refreshes the release's Compose and Caddy configuration, the
+`roomote` CLI itself, and the systemd unit together (operator override files
+and their registration are left untouched);
 mixing newer application images with an older Caddyfile can leave routes used
 by the new controller unavailable until the deployment configuration is also
 updated.

deploy/ci/validate-deployment-artifacts.mjs

@@ -17,6 +17,8 @@ const read = (path) => readFileSync(join(root, path), 'utf8');
 const catalog = JSON.parse(read('deploy/deployment-catalog.json'));
 const installer = read('deploy/install.sh');
 const deployer = read('deploy/scripts/deploy.sh');
+const upgrader = read('deploy/scripts/upgrade.sh');
+const hostCli = read('deploy/host/roomote');
 const productionEnvExample = read('.env.production.example');
 
 function fail(message) {
@@ -27,6 +29,49 @@ function assert(condition, message) {
   if (!condition) fail(message);
 }
 
+// The host CLI is the single owner of the Compose invocation (COMPOSE_FILE
+// registry in .env) and of the systemd unit. Every other surface must route
+// through it so operator overrides can never be dropped by a sibling path.
+assert(
+  hostCli.includes('read_env_value COMPOSE_FILE') &&
+    hostCli.includes('compose_args+=(-f "$install_root/$entry")'),
+  'host CLI: compose invocations must be built from the COMPOSE_FILE registry',
+);
+assert(
+  hostCli.includes('ExecStart=$cli_path up') &&
+    hostCli.includes('ExecStop=$cli_path down') &&
+    hostCli.includes('refresh_host_cli "$repo" "$fetch_ref"'),
+  'host CLI: it must own the systemd unit and refresh itself during upgrades',
+);
+assert(
+  installer.includes('/usr/local/bin/roomote sync-unit') &&
+    installer.includes('/usr/local/bin/roomote up') &&
+    !installer.includes('docker compose --env-file'),
+  'installer: systemd unit and stack start must go through the host CLI',
+);
+for (const [name, script] of [
+  ['deploy.sh', deployer],
+  ['upgrade.sh', upgrader],
+]) {
+  assert(
+    script.includes('/usr/local/bin/roomote sync-unit') &&
+      script.includes('/usr/local/bin/roomote up') &&
+      script.includes('/usr/local/bin/roomote docker pull') &&
+      !script.includes('docker compose --env-file') &&
+      !/^docker /m.test(script),
+    `managed ${name}: remote docker and compose operations must go through the host CLI`,
+  );
+}
+assert(
+  deployer.includes('for key in COMPOSE_FILE ROOMOTE_DOCKER_BIN'),
+  'managed deploy.sh: redeploys must carry forward host-owned .env state (override registry, Docker path)',
+);
+assert(
+  hostCli.includes('compose-overrides') &&
+    hostCli.includes("backup_services_stopped='false'"),
+  'host CLI: backups must stage override files and keep trap state global',
+);
+
 assert(
   installer.includes('preview_domain="$domain"') &&
     installer.includes(
@@ -610,7 +655,12 @@ for (const script of [
   'deploy/ci/deployment-smoke.sh',
   'deploy/ci/upgrade-compatibility.sh',
   'deploy/host/tests/backup-restore.integration.sh',
+  'deploy/host/tests/backup-failed-restart.sh',
+  'deploy/host/tests/compose-docker-path.sh',
+  'deploy/host/tests/compose-overrides.sh',
+  'deploy/host/tests/restore-failed-cleanup.sh',
   'deploy/host/tests/upgrade-failed-pull.sh',
+  'deploy/host/tests/upgrade-refresh.sh',
   '.docker/gbrain/entrypoint.sh',
 ]) {
   execFileSync('bash', ['-n', join(root, script)], { stdio: 'pipe' });