diff --git a/.github/steps/build-binary/action.yml b/.github/steps/build-binary/action.yml index f9c38cd2..749992ec 100644 --- a/.github/steps/build-binary/action.yml +++ b/.github/steps/build-binary/action.yml @@ -26,7 +26,7 @@ runs: ${BUILD_PROFILE_FLAG} \ --target "${{ inputs.target }}" \ --package "${{ inputs.package-name }}" \ - --bin "${{ inputs.artifact-name }}" \ + --bins \ ${FEATURE_ARGS} elif [[ "${{ inputs.artifact-kind }}" == "cdylib" ]]; then cargo build \ diff --git a/.github/steps/prepare-debian-package-assets/action.yml b/.github/steps/prepare-debian-package-assets/action.yml index 5aae0515..b0b60f48 100644 --- a/.github/steps/prepare-debian-package-assets/action.yml +++ b/.github/steps/prepare-debian-package-assets/action.yml @@ -32,6 +32,26 @@ runs: cp "$PACKAGE_DIR/${SBOM_BASENAME}.json" "${{ inputs.package-root }}/debian/sbom.json" cp "$PACKAGE_DIR/${SBOM_BASENAME}.xml" "${{ inputs.package-root }}/debian/sbom.xml" + # Copy any additional binaries built alongside the primary artifact. + # These are binaries defined in src/bin/ of the package that are not + # the primary artifact. cargo-deb expects them in target/release/. + PKG_BIN_DIR="${{ inputs.package-root }}/src/bin" + if [[ -d "$PKG_BIN_DIR" ]]; then + for RS_FILE in "$PKG_BIN_DIR"/*.rs; do + [[ -f "$RS_FILE" ]] || continue + BIN_NAME="$(basename "$RS_FILE" .rs)" + if [[ "$BIN_NAME" == "$ARTIFACT_BASENAME" ]]; then + continue + fi + BIN_SRC="target/${{ inputs.target }}/${PROFILE}/${BIN_NAME}" + if [[ -f "$BIN_SRC" ]]; then + cp "$BIN_SRC" "target/release/${BIN_NAME}" + chmod +x "target/release/${BIN_NAME}" + echo "Copied additional binary: ${BIN_NAME}" + fi + done + fi + - name: Copy generated man page and compress Debian assets shell: bash run: | diff --git a/Cargo.lock b/Cargo.lock index 4aa85a70..73f5f86d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,7 +98,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -158,15 +158,15 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64" -version = "0.22.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" [[package]] name = "bitflags" @@ -200,15 +200,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.2.65" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "shlex", @@ -222,9 +222,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cpufeatures", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.2" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -255,14 +255,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -301,15 +301,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "console-api" version = "0.9.0" @@ -376,18 +367,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crypto-common" @@ -400,9 +391,9 @@ dependencies = [ [[package]] name = "defmt" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -410,15 +401,14 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" dependencies = [ "defmt-parser", - "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -449,14 +439,14 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "env_filter" @@ -499,11 +489,10 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -520,9 +509,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" @@ -647,7 +636,7 @@ checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -727,11 +716,11 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hdrhistogram" -version = "7.5.4" +version = "7.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +checksum = "f49d1053f4708f0af3cf9fc5bffc7e68a914a3c45becb231c80068c9c3f78bea" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "byteorder", "flate2", "nom", @@ -756,9 +745,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -766,9 +755,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -791,24 +780,24 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -974,9 +963,9 @@ dependencies = [ [[package]] name = "inotify" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533e68a5842e734946fe159fb03fc9bbbb254f590dd0d8ad321ae5ff7beca2c1" +checksum = "153be1941a183ec9ccd095ddbe17a8b8d435ef6c76e9e02451b933c3999af2c8" dependencies = [ "bitflags 2.13.1", "inotify-sys", @@ -985,9 +974,9 @@ dependencies = [ [[package]] name = "inotify-sys" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea94e891b3606826e9c998be69ddca42247dad8ad50b1649a5cb7e1c9ae06fd" +checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d" dependencies = [ "libc", ] @@ -1015,11 +1004,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.29" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ "defmt", + "jiff-core", "jiff-static", "log", "portable-atomic", @@ -1027,15 +1017,25 @@ dependencies = [ "serde_core", ] +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + [[package]] name = "jiff-static" -version = "0.2.29" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ + "jiff-core", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1066,9 +1066,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libseccomp" @@ -1142,9 +1142,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mime" @@ -1152,12 +1152,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1170,9 +1164,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "log", @@ -1188,12 +1182,11 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "nom" -version = "7.1.3" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" dependencies = [ "memchr", - "minimal-lexical", ] [[package]] @@ -1225,9 +1218,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" dependencies = [ "num-integer", "num-traits", @@ -1270,7 +1263,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1307,7 +1300,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1385,7 +1378,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1402,9 +1395,9 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "portable-atomic-util" @@ -1431,7 +1424,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1443,33 +1436,11 @@ dependencies = [ "toml_edit", ] -[[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" -dependencies = [ - "proc-macro2", - "quote", -] - -[[package]] -name = "proc-macro-error2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" -dependencies = [ - "proc-macro-error-attr2", - "proc-macro2", - "quote", - "syn 2.0.118", -] - [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -1501,7 +1472,7 @@ dependencies = [ "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", - "syn 2.0.118", + "syn 2.0.119", "tempfile", ] @@ -1515,7 +1486,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1613,9 +1584,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -1677,9 +1648,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -1689,9 +1660,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -1706,9 +1677,9 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "regorus" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419a0413adeece71e4d4a64fb75adc359cb807496f0dfd10f429517be908b807" +checksum = "3cc4dc91481b1d4001ba7f2e81f7faf674142e0ac36d37d79e5f02764d06571e" dependencies = [ "anyhow", "lazy_static", @@ -1742,18 +1713,18 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "zeroize", ] [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -1803,7 +1774,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -1818,9 +1789,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -1871,9 +1842,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "slab" @@ -1889,9 +1860,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -1899,9 +1870,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.10.0" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +checksum = "8abadc99fd9c7bbb7d0ca2b31d72a067d0c0dcd7aad25ab8cac71ba91417694b" [[package]] name = "stable_deref_trait" @@ -1917,9 +1888,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -1928,9 +1899,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -1951,7 +1922,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1971,6 +1942,7 @@ dependencies = [ "tokio", "tokio-stream", "tonic", + "tonic-health", ] [[package]] @@ -1984,6 +1956,7 @@ dependencies = [ "protoc-bin-vendored", "tokio", "tonic", + "tonic-health", "tonic-prost", "tonic-prost-build", "tower", @@ -2030,12 +2003,14 @@ name = "tacacsrs-agentd" version = "0.0.0-dev" dependencies = [ "anyhow", + "async-trait", "clap", "clap_mangen", "console-subscriber", "env_logger", "futures-util", "log", + "rand", "rustls-pki-types", "tacacsrs-agent", "tacacsrs-agent-client", @@ -2045,6 +2020,10 @@ dependencies = [ "tacacsrs-networking", "tacacsrs-sonic", "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tonic-health", ] [[package]] @@ -2063,7 +2042,7 @@ version = "0.0.0-dev" dependencies = [ "anyhow", "async-trait", - "base64 0.22.1", + "base64 0.23.0", "futures-core", "futures-util", "log", @@ -2080,7 +2059,7 @@ name = "tacacsrs-config" version = "0.0.0-dev" dependencies = [ "anyhow", - "base64 0.22.1", + "base64 0.23.0", "bitflags 2.13.1", "serde", "serde_ignored", @@ -2208,29 +2187,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] @@ -2247,9 +2226,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.0" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -2265,13 +2244,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2287,9 +2266,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -2299,13 +2278,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -2321,9 +2301,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.12+spec-1.1.0" +version = "0.25.13+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ "indexmap", "toml_datetime", @@ -2333,9 +2313,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] @@ -2378,7 +2358,20 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", +] + +[[package]] +name = "tonic-health" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcfab99db777fba2802f0dfa861d1628d1ae916fb199d29819941f139ae85082" +dependencies = [ + "prost", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", ] [[package]] @@ -2403,7 +2396,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.118", + "syn 2.0.119", "tempfile", "tonic-build", ] @@ -2458,7 +2451,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2671,9 +2664,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" dependencies = [ "memchr", ] @@ -2686,9 +2679,9 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "xxhash-rust" -version = "0.8.15" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yoke" @@ -2709,7 +2702,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -2730,7 +2723,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -2770,11 +2763,11 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/docs/sonic-agentd-container.md b/docs/sonic-agentd-container.md index 665c4612..c351ce1e 100644 --- a/docs/sonic-agentd-container.md +++ b/docs/sonic-agentd-container.md @@ -20,10 +20,12 @@ is the recommended mode for this agent. ```bash docker run --rm --network host \ -v /var/run/redis/redis.sock:/var/run/redis/redis.sock \ + -v /run/tacacs:/run/tacacs \ -p 127.0.0.1:49:49 \ ghcr.io/authscaffold/tacacsrs-agentd:2026.706.1 \ --sonic \ --service-mode both \ + --host-integration none \ --proxy-endpoint 127.0.0.1:49 \ -vv ``` @@ -37,7 +39,9 @@ The flags mean: | `-p 127.0.0.1:49:49` | Harmless with host networking, but ignored by Docker because there is no separate container network namespace to publish from. | | `--sonic` | Load TACACS+ server configuration from SONiC CONFIG_DB, database `4`. | | `--service-mode both` | Run both the local client API and the raw TACACS+ proxy service. | +| `--host-integration none` | Disable systemd notification inside the container. | | `--proxy-endpoint 127.0.0.1:49` | Bind the proxy on loopback only, so local SONiC clients can connect without exposing port `49` on external interfaces. | +| `-v /run/tacacs:/run/tacacs` | Expose the Client API UDS and standard gRPC health service to host consumers and exec probes. | | `-vv` | Enable info-level logging. | Because `-p` is ignored in host networking mode, the command can also be written @@ -46,9 +50,11 @@ without the publish flag: ```bash docker run --rm --network host \ -v /var/run/redis/redis.sock:/var/run/redis/redis.sock \ + -v /run/tacacs:/run/tacacs \ ghcr.io/authscaffold/tacacsrs-agentd:2026.706.1 \ --sonic \ --service-mode both \ + --host-integration none \ --proxy-endpoint 127.0.0.1:49 \ -vv ``` @@ -114,14 +120,22 @@ At least one `TACPLUS_SERVER` row must be present for upstream TACACS+ traffic. For the schema mapping and Redis notification details, see [SONiC ConfigDB Integration](sonic-configdb-integration.md). +The process and Client API listener start even if Redis is temporarily unavailable. Startup and readiness remain not serving while the daemon retries ConfigDB with capped jittered backoff. A valid snapshot makes the same process ready without a restart. Upstream reachability does not gate readiness. + +Use the standard exec probe in the image: + +```bash +tacacsrs-agent-health \ + --endpoint /run/tacacs/tacacs.sock \ + --check readiness \ + --timeout-seconds 2 +``` + The startup log should include lines similar to: ```text Configured SONiC ConfigDB datastore: url='unix:///var/run/redis/redis.sock?db=4', db=4 -Initial configuration loaded from datastore 'sonic-configdb' -Upstream servers: 1 configured, probe interval: 30s TACACS+ proxy endpoint: Tcp(127.0.0.1:49) ``` -If CONFIG_DB changes should be picked up live, enable Redis keyspace -notifications as described in the ConfigDB integration guide. \ No newline at end of file +Enable Redis keyspace notifications as described in the ConfigDB integration guide. If the subscription fails or ends, the daemon retains its known-good configuration, reloads to cover missed events, and resubscribes. \ No newline at end of file diff --git a/docs/sonic-configdb-integration.md b/docs/sonic-configdb-integration.md index 882f8185..ce565323 100644 --- a/docs/sonic-configdb-integration.md +++ b/docs/sonic-configdb-integration.md @@ -252,25 +252,22 @@ cargo run -p tacacsrs-agentd -- \ ``` Mutating the Redis rows should produce the documented configuration-change log -message. The current daemon observes changes but does not hot-swap upstream -connections in place; restart `tacacsrs-agentd` to apply a changed server set. +message. The daemon atomically applies each valid filtered snapshot to new +sessions without restarting. In-flight sessions keep their existing server-set +snapshot and connection handles. ## Hot reload behaviour -When a TACPLUS-prefixed key changes in CONFIG_DB, the bridge: +When a TACPLUS-prefixed key changes in CONFIG_DB, the runtime: 1. Coalesces additional changes that arrive within a short debounce window. 2. Re-reads the full TACPLUS / TACPLUS_SERVER tables. 3. Validates the new snapshot against the YANG schema. -4. Emits a `ConfigChange` event carrying the new snapshot and a delta - describing which servers were added, removed, or modified. - -The current daemon does not yet hot-swap upstream connections in place: it -records the change and asks the operator to restart the service to apply -the new configuration. This is the smallest safe step that keeps in-flight -TACACS+ sessions intact, and the plumbing is shaped so that a future -implementation can replace the listener body with an atomic state-rebuild -without changing the surrounding lifecycle or the datastore contract. +4. Emits a typed changed or rejected event. +5. Filters proxy self-loops and validates the complete candidate. +6. Atomically replaces the server set for new sessions while preserving unchanged cached connections. + +Invalid candidates leave the previous known-good configuration active and mark runtime health stale/degraded. If Redis is unavailable at process startup, enabled listeners still bind and liveness serves while startup/readiness remain not serving. The daemon retries with capped jittered backoff. Subscription setup failures and ended streams mark the snapshot stale, trigger a fresh load to cover missed changes, and resubscribe without exiting. ## Operational commands diff --git a/docs/tacacs-plus-tls-transition.md b/docs/tacacs-plus-tls-transition.md index a2928d06..55054b5d 100644 --- a/docs/tacacs-plus-tls-transition.md +++ b/docs/tacacs-plus-tls-transition.md @@ -53,13 +53,12 @@ Keep these in the legacy client configuration: ## Shared Secret Compatibility -Proxy mode currently uses the selected upstream server's TACACS+ shared secret -for the downstream client connection too. That means: +The downstream client hop has its own TACACS+ obfuscation policy and does not follow the selected upstream server: -- Keep the legacy client `secret=` value and the `tacacsrs-agentd --shared-secret` value aligned. -- Use the same TACACS+ shared secret across the upstream servers used by one proxy deployment. -- Avoid a cutover where the local client secret differs from the upstream server secret unless the proxy gains a separate downstream-secret setting. -- If the upstream TLS server disables TACACS+ packet obfuscation entirely, verify whether the legacy client can send unencrypted TACACS+ packets before relying on proxy mode. +- Outside SONiC mode, keep the legacy client `secret=` value aligned with `tacacsrs-agentd --proxy-shared-secret`. +- In SONiC mode, configure the local loopback proxy row with the legacy client secret. The daemon filters that self-targeting row from the upstream set and uses its resolved `passkey` for downstream obfuscation. +- If neither source provides a downstream secret, verify that the legacy client can send TACACS+ packets with the unencrypted flag. +- Upstream servers may use different classic shared secrets or TLS credentials; failover does not change the local client secret. TLS protects the upstream connection, but it does not replace the downstream TACACS+ packet format expected by `pam_tacplus` or `audisp-tacplus`. diff --git a/docs/tacacsrs-agentd.md b/docs/tacacsrs-agentd.md index bcaa1f4d..8e40e15b 100644 --- a/docs/tacacsrs-agentd.md +++ b/docs/tacacsrs-agentd.md @@ -61,6 +61,7 @@ tacon --service-endpoint /run/tacacs/tacacs.sock \ | `--proxy-endpoint ` | *(disabled)* | Optional TACACS+ proxy listener on a Unix socket path or loopback TCP address | | `--service-mode ` | `client-api`, or `both` when `--proxy-endpoint` is set | Runtime services to host: `client-api`, `tacacs-proxy`, or `both` | | `--socket-mode ` | `660` | File permission mode for the Unix socket (octal) | +| `--host-integration ` | `auto` | Host adapter: `auto`, `none`, or strict `systemd` | ### Runtime Service Modes @@ -92,7 +93,7 @@ Proxy mode is deliberately packet-transparent: Proxy TCP endpoints must be loopback addresses. Unix domain socket endpoints use the same `--socket-mode` value as the IPC listener. When `client-api` and `tacacs-proxy` run together, the proxy endpoint must be different from `--listen-endpoint`. -The downstream TACACS+ shared-secret behavior follows the upstream server selected for that connection. If the selected upstream server has a `shared-secret`, the proxy uses that secret to deobfuscate downstream packets and obfuscate replies. If the selected upstream server has no shared secret, downstream packets must be sent with the unencrypted flag. When configured upstream servers use different shared secrets, a reconnect or failover can select a server with a different downstream secret; keep upstream shared secrets identical when using proxy mode. +Downstream TACACS+ obfuscation is independent of the selected upstream transport. In SONiC mode, the daemon filters rows that target its own loopback proxy and uses the highest-priority matching row's resolved `passkey` for the local hop. Outside SONiC mode, use `--proxy-shared-secret`. If neither source provides a local-hop secret, downstream clients must send unobfuscated TACACS+ packets. Upstream failover never changes the downstream secret. ### Upstream Encryption @@ -173,6 +174,44 @@ When multiple IPC requests arrive simultaneously during a reconnect, only one co On startup the daemon attempts to connect to servers in order and stops at the first success. This prevents connection storms when many instances start simultaneously (e.g. during a fleet rollout). If no server is reachable at startup, the daemon still starts and requests will retry on demand. +SONiC ConfigDB is supervised differently from local CLI or file input. The daemon binds enabled listeners with an empty runtime configuration, reports startup/readiness as not serving, and retries ConfigDB with capped jittered backoff. When Redis becomes available, the same process applies the first valid snapshot and becomes ready. Subscription failures and ended streams trigger a fresh load before resubscription; invalid candidates leave the previous known-good runtime configuration active and mark health degraded. + +## Health and Probes + +The Client API endpoint also serves the standard `grpc.health.v1.Health` protocol. No custom health protobuf or additional network listener is used. + +| Service name | Meaning | +|---|---| +| `tacacsrs.agent.health.v1.Startup` | A validated snapshot is applied and every enabled listener is bound | +| `tacacsrs.agent.health.v1.Liveness` | The process is starting or serving and the Client API can answer | +| `tacacsrs.agent.health.v1.Readiness` | Startup is complete and at least one eligible upstream is configured | +| *(empty service name)* | Same as readiness | +| `tacacsrs.agent.v1.TacacsAgent` | Same as readiness for the business RPC service | + +Current upstream reachability is diagnostic and does not gate readiness or liveness. During shutdown all names become `NOT_SERVING` before listeners stop accepting work. + +Use the packaged probe for Kubernetes exec probes and local diagnostics: + +```bash +tacacsrs-agent-health \ + --endpoint /run/tacacs/tacacs.sock \ + --check readiness \ + --timeout-seconds 2 +``` + +| Exit code | Meaning | +|---|---| +| `0` | The selected health name is `SERVING` | +| `1` | The selected health name is not serving or unknown | +| `2` | Probe invocation or endpoint syntax is invalid | +| `3` | Endpoint, timeout, transport, or protocol failure | + +The health endpoint exists only when `client-api` is enabled. A proxy-only process still publishes runtime state to logs and systemd but cannot use this gRPC probe. + +## Host Integration + +`--host-integration auto` selects systemd only when `NOTIFY_SOCKET` is present. `none` never invokes a host API and is the container setting. Explicit `systemd` requires both `NOTIFY_SOCKET` and `systemd-notify`; missing prerequisites and runtime notification failures are fatal. Systemd receives a waiting status before readiness, `READY=1` exactly once, sanitized degraded status updates, and `STOPPING=1` before listener drain. + ## IPC Protocol The daemon communicates with clients via gRPC over Unix domain sockets (Linux) or loopback TCP (other platforms). The protocol is defined in protobuf: @@ -199,12 +238,14 @@ After=network-online.target Wants=network-online.target [Service] -Type=simple +Type=notify +NotifyAccess=all ExecStart=/usr/local/bin/tacacsrs-agentd \ --server-addr tacacs1.example.com:49 \ --server-addr tacacs2.example.com:49 \ --listen-endpoint /run/tacacs/tacacs.sock \ --socket-mode 660 \ + --host-integration systemd \ --shared-secret "shared_secret" \ --preferred-probe-interval-seconds 30 Restart=on-failure diff --git a/executables/tacacsrs_agentd/Cargo.toml b/executables/tacacsrs_agentd/Cargo.toml index e78a7420..a63009c0 100644 --- a/executables/tacacsrs_agentd/Cargo.toml +++ b/executables/tacacsrs_agentd/Cargo.toml @@ -8,11 +8,13 @@ description = "Central TACACS+ client service for local consumers" [dependencies] anyhow = { version = "1.0.102", features = ["backtrace"] } +async-trait = "0.1.89" clap = { version = "4.6.1", features = ["derive"] } console-subscriber = { version = "0.5.0", optional = true } env_logger = "0.11.10" futures-util = { version = "0.3", default-features = false } log = "0.4" +rand = "0.10.0" rustls-pki-types = "1.14.1" tacacsrs-agent-client = { path = "../../libraries/tacacsrs_agent_client" } tacacsrs-agent = { path = "../../libraries/tacacsrs_agent" } @@ -22,11 +24,17 @@ tacacsrs-datastore = { path = "../../libraries/tacacsrs_datastore" } tacacsrs-networking = { path = "../../libraries/tacacsrs_networking" } tacacsrs-sonic = { path = "../../libraries/tacacsrs_sonic" } tokio = { version = "1.52.3", features = ["full"] } +tokio-stream = "0.1.18" +tokio-util = "0.7.18" +tonic-health = "0.14.6" [features] default = [] console = ["console-subscriber"] +[dev-dependencies] +tonic = "0.14.6" + [build-dependencies] clap = { version = "4.6.1", features = ["derive"] } clap_mangen = "0.3.0" @@ -54,6 +62,7 @@ priority = "optional" maintainer-scripts = "debian/maintainer-scripts" assets = [ ["target/release/tacacsrs-agentd", "usr/sbin/", "755"], + ["target/release/tacacsrs-agent-health", "usr/bin/", "755"], ["debian/tacacsrs-agentd-config.ini", "etc/tacacsrs-agentd/config.ini", "644"], ["sonic/tacacsrs-agentd.service", "lib/systemd/system/tacacsrs-agentd.service", "644"], ["../../docs/tacacsrs-agentd.md", "usr/share/doc/tacacsrs-agentd/README.md", "644"], diff --git a/executables/tacacsrs_agentd/README.md b/executables/tacacsrs_agentd/README.md index 6d7d5506..b00f46a1 100644 --- a/executables/tacacsrs_agentd/README.md +++ b/executables/tacacsrs_agentd/README.md @@ -4,6 +4,21 @@ For full deployment and failover guidance, see [../../docs/tacacsrs-agentd.md](../../docs/tacacsrs-agentd.md). +## Health and Host Integration + +When the Client API is enabled, its existing endpoint also serves standard `grpc.health.v1.Health` names for startup, liveness, readiness, the overall empty name, and `tacacsrs.agent.v1.TacacsAgent`. Use the packaged probe: + +```bash +tacacsrs-agent-health \ + --endpoint /run/tacacs/tacacs.sock \ + --check readiness \ + --timeout-seconds 2 +``` + +Use `--host-integration none` in containers. `auto` selects systemd only when `NOTIFY_SOCKET` is present. Explicit `systemd` requires the notification socket and helper, sends `READY=1` once, and sends `STOPPING=1` before listener drain. + +SONiC ConfigDB startup and notification subscriptions are supervised. The daemon may bind listeners before Redis exists, retries with capped jittered backoff, applies valid snapshots without restarting, and retains the previous known-good configuration after invalid reloads or subscription outages. + ## Runtime Service Modes Use `--service-mode client-api`, `--service-mode tacacs-proxy`, or diff --git a/executables/tacacsrs_agentd/sonic/tacacsrs-agentd.service b/executables/tacacsrs_agentd/sonic/tacacsrs-agentd.service index 480b1f50..1350bfa4 100644 --- a/executables/tacacsrs_agentd/sonic/tacacsrs-agentd.service +++ b/executables/tacacsrs_agentd/sonic/tacacsrs-agentd.service @@ -1,11 +1,10 @@ [Unit] Description=Central TACACS+ client service for local consumers (SONiC) Documentation=https://github.com/AuthScaffold/tacacs-rs -# SONiC stores configuration in Redis (CONFIG_DB). Wait for the database -# service so that ConfigDB is ready before we attempt to read TACPLUS tables. -Requires=database.service +# SONiC stores configuration in Redis (CONFIG_DB). Ordering normally avoids an +# initial retry, but the daemon also survives database startup and restarts. After=database.service network-online.target -Wants=network-online.target +Wants=database.service network-online.target [Service] # Keep the unit in "activating" with a clear status line until SONiC pushes @@ -17,6 +16,7 @@ ExecStart=/usr/sbin/tacacsrs-agentd \ --sonic \ --listen-endpoint /run/tacacs/tacacs.sock \ --socket-mode 666 \ + --host-integration systemd \ -vv # Recreate the socket after restart and tear it down on stop. @@ -28,8 +28,8 @@ TimeoutStartSec=0 TimeoutStopSec=15s # SONiC keyspace notifications must be enabled on Redis for hot reload to -# work; the agent will start without them but will only react to changes on -# restart. +# work. The agent retries subscription setup and reloads before resubscribing +# after a notification stream ends. # redis-cli -n 4 CONFIG SET notify-keyspace-events KEA # Hardening (loose defaults compatible with SONiC's privileged service model; diff --git a/executables/tacacsrs_agentd/src/bin/tacacsrs-agent-health.rs b/executables/tacacsrs_agentd/src/bin/tacacsrs-agent-health.rs new file mode 100644 index 00000000..d2a08765 --- /dev/null +++ b/executables/tacacsrs_agentd/src/bin/tacacsrs-agent-health.rs @@ -0,0 +1,131 @@ +//! Exec probe for the agent's standard gRPC health service. + +use std::process::ExitCode; +use std::str::FromStr; +use std::time::Duration; + +use clap::{Parser, ValueEnum}; +use tacacsrs_agent_client::health::{ + LIVENESS_HEALTH_SERVICE, READINESS_HEALTH_SERVICE, STARTUP_HEALTH_SERVICE, +}; +use tacacsrs_agent_client::{HealthClient, IpcEndpoint}; +use tonic_health::pb::health_check_response::ServingStatus; + +const EXIT_NOT_SERVING: u8 = 1; +const EXIT_INVOCATION_ERROR: u8 = 2; +const EXIT_CHECK_ERROR: u8 = 3; + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum HealthCheck { + Startup, + Liveness, + Readiness, +} + +impl HealthCheck { + const fn service_name(self) -> &'static str { + match self { + Self::Startup => STARTUP_HEALTH_SERVICE, + Self::Liveness => LIVENESS_HEALTH_SERVICE, + Self::Readiness => READINESS_HEALTH_SERVICE, + } + } +} + +#[derive(Debug, Parser)] +#[command(name = "tacacsrs-agent-health", version, author)] +#[command(about = "Check tacacsrs-agentd startup, liveness, or readiness")] +struct Cli { + /// Client API Unix socket path or loopback TCP endpoint. + #[arg(long, default_value = "/run/tacacs/tacacs.sock")] + endpoint: String, + + /// Health view to check. + #[arg(long, value_enum)] + check: HealthCheck, + + /// Maximum total check duration. + #[arg(long, default_value_t = 2, value_parser = clap::value_parser!(u64).range(1..))] + timeout_seconds: u64, +} + +#[tokio::main] +async fn main() -> ExitCode { + let cli = match Cli::try_parse() { + Ok(cli) => cli, + Err(error) => { + let _ = error.print(); + return ExitCode::from(EXIT_INVOCATION_ERROR); + } + }; + let Ok(endpoint) = IpcEndpoint::from_str(&cli.endpoint) else { + eprintln!("invalid health endpoint: {}", cli.endpoint); + return ExitCode::from(EXIT_INVOCATION_ERROR); + }; + + match tokio::time::timeout( + Duration::from_secs(cli.timeout_seconds), + check_health(&endpoint, cli.check), + ) + .await + { + Ok(Ok(ServingStatus::Serving)) => ExitCode::SUCCESS, + Ok(Ok(ServingStatus::NotServing | ServingStatus::Unknown)) => { + eprintln!("{} health check is not serving", cli.check.service_name()); + ExitCode::from(EXIT_NOT_SERVING) + } + Ok(Err(())) => { + eprintln!("health check failed for endpoint {}", cli.endpoint); + ExitCode::from(EXIT_CHECK_ERROR) + } + Err(_) => { + eprintln!("health check timed out for endpoint {}", cli.endpoint); + ExitCode::from(EXIT_CHECK_ERROR) + } + Ok(Ok(ServingStatus::ServiceUnknown)) => { + eprintln!("{} health check is unknown", cli.check.service_name()); + ExitCode::from(EXIT_NOT_SERVING) + } + } +} + +async fn check_health(endpoint: &IpcEndpoint, check: HealthCheck) -> Result { + let mut client = HealthClient::connect(endpoint).await.map_err(|_| ())?; + client.check(check.service_name()).await.map_err(|_| ()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cli_accepts_documented_contract() { + let cli = Cli::try_parse_from([ + "tacacsrs-agent-health", + "--endpoint", + "127.0.0.1:9049", + "--check", + "readiness", + "--timeout-seconds", + "2", + ]) + .expect("documented CLI should parse"); + + assert_eq!(cli.endpoint, "127.0.0.1:9049"); + assert_eq!(cli.check.service_name(), READINESS_HEALTH_SERVICE); + assert_eq!(cli.timeout_seconds, 2); + } + + #[test] + fn cli_rejects_zero_timeout() { + let result = Cli::try_parse_from([ + "tacacsrs-agent-health", + "--check", + "liveness", + "--timeout-seconds", + "0", + ]); + + assert!(result.is_err()); + } +} diff --git a/executables/tacacsrs_agentd/src/cli.rs b/executables/tacacsrs_agentd/src/cli.rs index b8c329b4..922cef9e 100644 --- a/executables/tacacsrs_agentd/src/cli.rs +++ b/executables/tacacsrs_agentd/src/cli.rs @@ -27,6 +27,19 @@ pub(crate) enum PskKeyExchange { PskOnly, } +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq, ValueEnum)] +pub(crate) enum HostIntegrationMode { + /// Detect systemd from `NOTIFY_SOCKET`; otherwise use no host integration. + #[default] + Auto, + + /// Disable all host-specific notifications. + None, + + /// Require and publish systemd notifications. + Systemd, +} + fn psk_dhe_ke_supported_group_parser( ) -> impl clap::builder::TypedValueParser { clap::builder::PossibleValuesParser::new(PskDheKeSupportedGroup::ALLOWED_VALUES.iter().copied()) @@ -47,6 +60,10 @@ fn psk_dhe_ke_supported_group_parser( .args(["config", "server_addresses", "sonic"]) ))] pub(crate) struct Cli { + /// Host process-manager integration mode. + #[arg(long, value_enum, default_value_t)] + pub(crate) host_integration: HostIntegrationMode, + /// Path to a YANG JSON configuration file (ietf-system-tacacs-plus). #[arg(long, value_name = "FILE", conflicts_with_all = [ "server_addresses", "shared_secret", "use_tls", diff --git a/executables/tacacsrs_agentd/src/config_supervisor.rs b/executables/tacacsrs_agentd/src/config_supervisor.rs new file mode 100644 index 00000000..fef12dc4 --- /dev/null +++ b/executables/tacacsrs_agentd/src/config_supervisor.rs @@ -0,0 +1,721 @@ +//! Cancellable datastore loading and change-subscription supervision. +//! +//! The supervisor owns dependency recovery while the reusable agent runtime +//! owns the applied server set: +//! +//! ```text +//! load candidate -> daemon filter -> atomic service reload -> health current +//! | | +//! +-- failure -> retain known good ------+-> health stale/unavailable +//! ``` +//! +//! Datastore policy determines whether initial failure is fatal and whether a +//! change stream must be continuously restored. Stream restoration always +//! performs a fresh load before resubscribing so changes missed during an +//! outage cannot be silently skipped. + +use std::sync::Arc; +use std::time::Duration; + +use futures_util::StreamExt; +use tacacsrs_agent::{DatastoreState, DegradationReason, RuntimeHealthPublisher, TacacsClientService}; +use tacacsrs_config::TacacsPlus; +use tacacsrs_datastore::{ + ChangeNotificationMode, ConfigChangeEvent, ConfigDatastore, InitialLoadPolicy, +}; +use tokio_util::sync::CancellationToken; + +use crate::config_filter::TacacsPlusFilter; + +/// Supplies retry delays without owning asynchronous sleeping. +pub(crate) trait RetryBackoff: Send + Sync { + /// Returns the delay after a consecutive failure, starting at attempt one. + fn delay(&self, attempt: u32) -> Duration; +} + +/// Capped exponential retry with bounded downward jitter. +#[derive(Debug)] +pub(crate) struct ExponentialBackoff { + initial: Duration, + maximum: Duration, +} + +impl ExponentialBackoff { + /// Creates production datastore retry timing. + #[must_use] + pub(crate) const fn production() -> Self { + Self { + initial: Duration::from_millis(250), + maximum: Duration::from_secs(30), + } + } +} + +impl RetryBackoff for ExponentialBackoff { + fn delay(&self, attempt: u32) -> Duration { + let exponent = attempt.saturating_sub(1).min(16); + let multiplier = 1u32 << exponent; + let capped = self.initial.saturating_mul(multiplier).min(self.maximum); + let millis = u64::try_from(capped.as_millis()).unwrap_or(u64::MAX); + let jitter_span = millis / 5; + if jitter_span == 0 { + return capped; + } + + let jitter = rand::random::() % jitter_span.saturating_add(1); + Duration::from_millis(millis.saturating_sub(jitter)) + } +} + +/// Applies and continuously refreshes one runtime datastore. +pub(crate) struct ConfigSupervisor { + datastore: Arc, + service: Arc, + health: RuntimeHealthPublisher, + config_filter: Arc, + backoff: Arc, +} + +impl ConfigSupervisor { + /// Creates a supervisor with production retry timing. + #[must_use] + pub(crate) fn new( + datastore: Arc, + service: Arc, + health: RuntimeHealthPublisher, + config_filter: Arc, + ) -> Self { + Self::with_backoff( + datastore, + service, + health, + config_filter, + Arc::new(ExponentialBackoff::production()), + ) + } + + fn with_backoff( + datastore: Arc, + service: Arc, + health: RuntimeHealthPublisher, + config_filter: Arc, + backoff: Arc, + ) -> Self { + Self { + datastore, + service, + health, + config_filter, + backoff, + } + } + + /// Loads and applies the initial snapshot according to datastore policy. + /// + /// # Errors + /// + /// Returns a sanitized error when a fail-fast datastore cannot supply a + /// valid candidate. Cancellation is a successful shutdown outcome. + pub(crate) async fn load_initial( + &self, + cancellation: &CancellationToken, + ) -> anyhow::Result<()> { + let policy = self.datastore.runtime_policy().initial_load; + let mut attempt = 0u32; + + loop { + self.health.set_datastore(DatastoreState::Loading); + let result = tokio::select! { + () = cancellation.cancelled() => return Ok(()), + result = self.load_and_apply() => result, + }; + + match result { + Ok(()) => { + self.mark_current(); + return Ok(()); + } + Err(()) if policy == InitialLoadPolicy::FailFast => { + self.mark_source_problem(); + anyhow::bail!( + "Datastore '{}' could not supply a valid initial configuration", + self.datastore.label(), + ); + } + Err(()) => { + self.mark_source_problem(); + attempt = attempt.saturating_add(1); + log::warn!( + "Datastore '{}' initial configuration is unavailable; retrying", + self.datastore.label(), + ); + if !self.wait_for_retry(attempt, cancellation).await { + return Ok(()); + } + } + } + } + } + + /// Runs initial loading and then maintains notifications until cancelled. + /// + /// # Errors + /// + /// Returns an error only when fail-fast initial loading fails. + pub(crate) async fn run(&self, cancellation: &CancellationToken) -> anyhow::Result<()> { + self.load_initial(cancellation).await?; + if cancellation.is_cancelled() { + return Ok(()); + } + self.run_notifications(cancellation).await; + Ok(()) + } + + /// Maintains a continuous notification source after initial load. + pub(crate) async fn run_notifications(&self, cancellation: &CancellationToken) { + if self.datastore.runtime_policy().change_notifications == ChangeNotificationMode::None { + cancellation.cancelled().await; + return; + } + + let mut attempt = 0u32; + let mut refresh_before_subscribe = false; + loop { + if refresh_before_subscribe { + if self.load_and_apply().await.is_ok() { + self.mark_current(); + attempt = 0; + } else { + self.mark_source_problem(); + attempt = attempt.saturating_add(1); + if !self.wait_for_retry(attempt, cancellation).await { + return; + } + continue; + } + } + + let subscription = tokio::select! { + () = cancellation.cancelled() => return, + result = self.datastore.subscribe() => result, + }; + let Ok(mut stream) = subscription else { + log::warn!( + "Datastore '{}' change subscription is unavailable", + self.datastore.label(), + ); + self.mark_subscription_unavailable(); + attempt = attempt.saturating_add(1); + if !self.wait_for_retry(attempt, cancellation).await { + return; + } + refresh_before_subscribe = true; + continue; + }; + + self.health.set_continuous_notifications_connected(true); + self.health + .set_degraded(DegradationReason::ChangeNotificationsUnavailable, false); + attempt = 0; + loop { + let event = tokio::select! { + () = cancellation.cancelled() => return, + event = stream.next() => event, + }; + match event { + Some(ConfigChangeEvent::Changed(change)) => { + if self.apply_candidate((*change.config).clone()).await.is_ok() { + self.mark_current(); + } else { + self.mark_candidate_rejected(); + } + } + Some(ConfigChangeEvent::CandidateRejected) => { + self.mark_candidate_rejected(); + } + None => { + log::warn!( + "Datastore '{}' continuous change stream ended; reconnecting", + self.datastore.label(), + ); + self.mark_subscription_unavailable(); + refresh_before_subscribe = true; + attempt = attempt.saturating_add(1); + if !self.wait_for_retry(attempt, cancellation).await { + return; + } + break; + } + } + } + } + } + + async fn load_and_apply(&self) -> Result<(), ()> { + let candidate = self.datastore.load().await.map_err(|_| { + log::warn!("Datastore '{}' configuration load failed", self.datastore.label()); + })?; + self.apply_candidate(candidate).await + } + + async fn apply_candidate(&self, candidate: TacacsPlus) -> Result<(), ()> { + let filtered = self.config_filter.filter(candidate).await.map_err(|_| { + log::warn!( + "Datastore '{}' configuration candidate was rejected by runtime filtering", + self.datastore.label(), + ); + })?; + self.service + .reload_tacacs_plus_with_proxy_downstream_obfuscation( + filtered.tacacs_plus, + filtered.proxy_downstream_obfuscation, + ) + .await + .map_err(|_| { + log::warn!( + "Datastore '{}' configuration candidate was rejected by runtime validation", + self.datastore.label(), + ); + }) + } + + async fn wait_for_retry(&self, attempt: u32, cancellation: &CancellationToken) -> bool { + let delay = self.backoff.delay(attempt); + tokio::select! { + () = cancellation.cancelled() => false, + () = tokio::time::sleep(delay) => true, + } + } + + fn mark_current(&self) { + self.health.set_datastore(DatastoreState::Current); + self.health + .set_degraded(DegradationReason::DatastoreUnavailable, false); + self.health + .set_degraded(DegradationReason::DatastoreStale, false); + self.health + .set_degraded(DegradationReason::CandidateConfigurationRejected, false); + } + + fn mark_source_problem(&self) { + let has_known_good = self.health.snapshot().has_applied_configuration(); + self.health.set_datastore(if has_known_good { + DatastoreState::Stale + } else { + DatastoreState::Unavailable + }); + self.health + .set_degraded(DegradationReason::DatastoreStale, has_known_good); + self.health + .set_degraded(DegradationReason::DatastoreUnavailable, !has_known_good); + } + + fn mark_subscription_unavailable(&self) { + self.health.set_continuous_notifications_connected(false); + self.health + .set_degraded(DegradationReason::ChangeNotificationsUnavailable, true); + self.mark_source_problem(); + } + + fn mark_candidate_rejected(&self) { + self.health + .set_degraded(DegradationReason::CandidateConfigurationRejected, true); + self.mark_source_problem(); + } +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use async_trait::async_trait; + use tacacsrs_agent::{EnabledServices, ProxyDownstreamObfuscation, ServiceConfig}; + use tacacsrs_agent_client::IpcEndpoint; + use tacacsrs_config::{TacacsPlusBuilder, TacacsPlusServerBuilder, TacacsPlusServerType}; + use tacacsrs_datastore::{ConfigChangeStream, DatastoreRuntimePolicy}; + use tokio::time::timeout; + + use super::*; + use crate::config_filter::NoopTacacsPlusFilter; + + #[derive(Debug, Clone, Copy)] + enum LoadStep { + Error, + Config(usize), + } + + #[derive(Debug, Clone, Copy)] + enum SubscriptionStep { + Error, + Empty, + Pending, + RejectedThenPending, + } + + struct ScriptedDatastore { + policy: DatastoreRuntimePolicy, + loads: Mutex>, + subscriptions: Mutex>, + load_count: AtomicUsize, + subscribe_count: AtomicUsize, + } + + impl ScriptedDatastore { + fn new( + policy: DatastoreRuntimePolicy, + loads: impl IntoIterator, + subscriptions: impl IntoIterator, + ) -> Self { + Self { + policy, + loads: Mutex::new(loads.into_iter().collect()), + subscriptions: Mutex::new(subscriptions.into_iter().collect()), + load_count: AtomicUsize::new(0), + subscribe_count: AtomicUsize::new(0), + } + } + } + + #[async_trait] + impl ConfigDatastore for ScriptedDatastore { + fn runtime_policy(&self) -> DatastoreRuntimePolicy { + self.policy + } + + async fn load(&self) -> anyhow::Result { + self.load_count.fetch_add(1, Ordering::Relaxed); + let step = self + .loads + .lock() + .expect("loads lock") + .pop_front() + .unwrap_or(LoadStep::Error); + match step { + LoadStep::Error => anyhow::bail!("scripted load failure"), + LoadStep::Config(server_count) => Ok(test_config(server_count)), + } + } + + async fn subscribe(&self) -> anyhow::Result { + self.subscribe_count.fetch_add(1, Ordering::Relaxed); + let step = self + .subscriptions + .lock() + .expect("subscriptions lock") + .pop_front() + .unwrap_or(SubscriptionStep::Pending); + match step { + SubscriptionStep::Error => anyhow::bail!("scripted subscription failure"), + SubscriptionStep::Empty => Ok(Box::pin(tokio_stream::empty())), + SubscriptionStep::Pending => Ok(Box::pin(tokio_stream::pending())), + SubscriptionStep::RejectedThenPending => Ok(Box::pin( + tokio_stream::once(ConfigChangeEvent::CandidateRejected) + .chain(tokio_stream::pending()), + )), + } + } + + fn label(&self) -> &'static str { + "scripted" + } + } + + #[derive(Debug)] + struct FixedBackoff(Duration); + + impl RetryBackoff for FixedBackoff { + fn delay(&self, _attempt: u32) -> Duration { + self.0 + } + } + + #[test] + fn exponential_backoff_stays_within_jitter_floor_and_maximum() { + let backoff = ExponentialBackoff { + initial: Duration::from_millis(250), + maximum: Duration::from_secs(30), + }; + + for attempt in 1..=24 { + let exponent = (attempt - 1).min(16); + let capped = Duration::from_millis(250) + .saturating_mul(1u32 << exponent) + .min(Duration::from_secs(30)); + let delay = backoff.delay(attempt); + + assert!(delay >= capped.saturating_mul(4) / 5); + assert!(delay <= capped); + } + } + + fn test_config(server_count: usize) -> TacacsPlus { + (0..server_count) + .fold(TacacsPlusBuilder::new(), |builder, index| { + builder.with_server_builder( + TacacsPlusServerBuilder::new( + format!("server-{index}"), + TacacsPlusServerType::all(), + format!("192.0.2.{}", index + 1), + 49, + ) + .with_shared_secret("test-secret"), + ) + }) + .build() + .expect("test config") + } + + fn supervisor( + datastore: Arc, + backoff: Duration, + ) -> (ConfigSupervisor, RuntimeHealthPublisher, Arc) { + let health = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + let service = Arc::new( + TacacsClientService::waiting_for_configuration( + ServiceConfig { + enabled_services: EnabledServices::CLIENT_API, + endpoint: IpcEndpoint::default_local(), + proxy_endpoint: None, + proxy_downstream_obfuscation: ProxyDownstreamObfuscation::default(), + tacacs_plus: TacacsPlus::empty(), + preferred_probe_interval: Duration::from_secs(1), + #[cfg(unix)] + socket_mode: 0o660, + disable_certificate_verification: false, + }, + health.clone(), + ) + .expect("service"), + ); + let supervisor = ConfigSupervisor::with_backoff( + datastore, + Arc::clone(&service), + health.clone(), + Arc::new(NoopTacacsPlusFilter::default()), + Arc::new(FixedBackoff(backoff)), + ); + (supervisor, health, service) + } + + #[tokio::test] + async fn fail_fast_returns_after_first_initial_load_failure() { + let datastore = Arc::new(ScriptedDatastore::new( + DatastoreRuntimePolicy::new(InitialLoadPolicy::FailFast, ChangeNotificationMode::None), + [LoadStep::Error], + [], + )); + let (supervisor, health, _) = supervisor(Arc::clone(&datastore), Duration::ZERO); + + let result = supervisor.load_initial(&CancellationToken::new()).await; + + assert!(result.is_err()); + let public_error = result + .expect_err("fail-fast should return an error") + .to_string(); + assert!(!public_error.contains("scripted load failure")); + assert!(!public_error.contains("test-secret")); + assert_eq!(datastore.load_count.load(Ordering::Relaxed), 1); + assert_eq!(health.snapshot().datastore(), DatastoreState::Unavailable); + } + + #[tokio::test] + async fn retrying_initial_load_applies_first_available_snapshot() { + let datastore = Arc::new(ScriptedDatastore::new( + DatastoreRuntimePolicy::new( + InitialLoadPolicy::RetryUntilAvailable, + ChangeNotificationMode::None, + ), + [LoadStep::Error, LoadStep::Error, LoadStep::Config(1)], + [], + )); + let (supervisor, health, service) = supervisor(Arc::clone(&datastore), Duration::ZERO); + + supervisor + .load_initial(&CancellationToken::new()) + .await + .expect("retry should recover"); + + assert_eq!(datastore.load_count.load(Ordering::Relaxed), 3); + assert_eq!(service.server_count(), 1); + assert!(health.snapshot().has_applied_configuration()); + assert_eq!(health.snapshot().datastore(), DatastoreState::Current); + } + + #[tokio::test] + async fn cancellation_interrupts_initial_retry_sleep() { + let datastore = Arc::new(ScriptedDatastore::new( + DatastoreRuntimePolicy::new( + InitialLoadPolicy::RetryUntilAvailable, + ChangeNotificationMode::None, + ), + [LoadStep::Error], + [], + )); + let (supervisor, _, _) = supervisor(Arc::clone(&datastore), Duration::MAX); + let cancellation = CancellationToken::new(); + let child = cancellation.clone(); + + let task = tokio::spawn(async move { supervisor.load_initial(&child).await }); + tokio::task::yield_now().await; + cancellation.cancel(); + + timeout(Duration::from_secs(1), task) + .await + .expect("supervisor should stop promptly") + .expect("task should join") + .expect("cancellation is successful"); + assert_eq!(datastore.load_count.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn notification_none_never_subscribes() { + let datastore = Arc::new(ScriptedDatastore::new( + DatastoreRuntimePolicy::new(InitialLoadPolicy::FailFast, ChangeNotificationMode::None), + [LoadStep::Config(1)], + [], + )); + let (supervisor, _, _) = supervisor(Arc::clone(&datastore), Duration::ZERO); + let cancellation = CancellationToken::new(); + cancellation.cancel(); + + supervisor + .run(&cancellation) + .await + .expect("static supervisor should stop"); + + assert_eq!(datastore.subscribe_count.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn ended_stream_refreshes_snapshot_before_resubscribing() { + let datastore = Arc::new(ScriptedDatastore::new( + DatastoreRuntimePolicy::new( + InitialLoadPolicy::RetryUntilAvailable, + ChangeNotificationMode::Continuous, + ), + [LoadStep::Config(1), LoadStep::Config(2)], + [SubscriptionStep::Empty, SubscriptionStep::Pending], + )); + let (supervisor, health, service) = supervisor(Arc::clone(&datastore), Duration::ZERO); + let cancellation = CancellationToken::new(); + let child = cancellation.clone(); + + let task = tokio::spawn(async move { supervisor.run(&child).await }); + timeout(Duration::from_secs(1), async { + while datastore.subscribe_count.load(Ordering::Relaxed) < 2 { + tokio::task::yield_now().await; + } + }) + .await + .expect("supervisor should resubscribe"); + cancellation.cancel(); + task.await + .expect("task should join") + .expect("supervisor should stop"); + + assert_eq!(service.server_count(), 2); + assert_eq!(health.snapshot().datastore(), DatastoreState::Current); + } + + #[tokio::test] + async fn rejected_candidate_retains_known_good_configuration() { + let datastore = Arc::new(ScriptedDatastore::new( + DatastoreRuntimePolicy::new( + InitialLoadPolicy::FailFast, + ChangeNotificationMode::Continuous, + ), + [LoadStep::Config(1)], + [SubscriptionStep::RejectedThenPending], + )); + let (supervisor, health, service) = supervisor(Arc::clone(&datastore), Duration::ZERO); + let cancellation = CancellationToken::new(); + let child = cancellation.clone(); + + supervisor + .load_initial(&cancellation) + .await + .expect("initial load"); + let task = tokio::spawn(async move { supervisor.run_notifications(&child).await }); + timeout(Duration::from_secs(1), async { + while !health + .snapshot() + .degradation_reasons() + .contains(&DegradationReason::CandidateConfigurationRejected) + { + tokio::task::yield_now().await; + } + }) + .await + .expect("rejection should be published"); + cancellation.cancel(); + task.await.expect("task should join"); + + assert_eq!(service.server_count(), 1); + assert_eq!(health.snapshot().datastore(), DatastoreState::Stale); + } + + #[tokio::test] + async fn subscription_setup_failure_refreshes_before_retry() { + let datastore = Arc::new(ScriptedDatastore::new( + DatastoreRuntimePolicy::new( + InitialLoadPolicy::RetryUntilAvailable, + ChangeNotificationMode::Continuous, + ), + [LoadStep::Config(1), LoadStep::Config(2)], + [SubscriptionStep::Error, SubscriptionStep::Pending], + )); + let (supervisor, _, service) = supervisor(Arc::clone(&datastore), Duration::ZERO); + let cancellation = CancellationToken::new(); + let child = cancellation.clone(); + + let task = tokio::spawn(async move { supervisor.run(&child).await }); + timeout(Duration::from_secs(1), async { + while datastore.subscribe_count.load(Ordering::Relaxed) < 2 { + tokio::task::yield_now().await; + } + }) + .await + .expect("supervisor should retry subscription"); + cancellation.cancel(); + task.await + .expect("task should join") + .expect("supervisor should stop"); + + assert_eq!(service.server_count(), 2); + } + + #[tokio::test] + async fn failed_refresh_after_stream_end_retries_load_before_resubscribing() { + let datastore = Arc::new(ScriptedDatastore::new( + DatastoreRuntimePolicy::new( + InitialLoadPolicy::RetryUntilAvailable, + ChangeNotificationMode::Continuous, + ), + [LoadStep::Config(1), LoadStep::Error, LoadStep::Config(2)], + [SubscriptionStep::Empty, SubscriptionStep::Pending], + )); + let (supervisor, health, service) = supervisor(Arc::clone(&datastore), Duration::ZERO); + let cancellation = CancellationToken::new(); + let child = cancellation.clone(); + + let task = tokio::spawn(async move { supervisor.run(&child).await }); + timeout(Duration::from_secs(1), async { + while datastore.subscribe_count.load(Ordering::Relaxed) < 2 { + tokio::task::yield_now().await; + } + }) + .await + .expect("supervisor should recover and resubscribe"); + cancellation.cancel(); + task.await + .expect("task should join") + .expect("supervisor should stop"); + + assert_eq!(datastore.load_count.load(Ordering::Relaxed), 3); + assert_eq!(datastore.subscribe_count.load(Ordering::Relaxed), 2); + assert_eq!(service.server_count(), 2); + assert_eq!(health.snapshot().datastore(), DatastoreState::Current); + } +} diff --git a/executables/tacacsrs_agentd/src/host_integration.rs b/executables/tacacsrs_agentd/src/host_integration.rs new file mode 100644 index 00000000..a854f9d2 --- /dev/null +++ b/executables/tacacsrs_agentd/src/host_integration.rs @@ -0,0 +1,347 @@ +//! Daemon-owned host process-manager integration. + +use std::process::Command; +use std::sync::Arc; + +use tacacsrs_agent::{RuntimeHealthSnapshot, RuntimeLifecycle}; +use tokio::sync::watch; +use tokio_util::sync::CancellationToken; + +use crate::cli::HostIntegrationMode; + +trait SystemdCommand: Send + Sync { + fn is_available(&self) -> bool; + fn execute(&self, arguments: &[String]) -> anyhow::Result<()>; +} + +#[derive(Debug)] +struct ProcessSystemdCommand; + +impl SystemdCommand for ProcessSystemdCommand { + fn is_available(&self) -> bool { + Command::new("systemd-notify") + .arg("--version") + .status() + .is_ok_and(|status| status.success()) + } + + fn execute(&self, arguments: &[String]) -> anyhow::Result<()> { + let status = Command::new("systemd-notify") + .args(arguments) + .status() + .map_err(|error| anyhow::anyhow!("failed to execute systemd-notify: {error}"))?; + if !status.success() { + anyhow::bail!("systemd-notify returned unsuccessful status: {status}"); + } + Ok(()) + } +} + +/// Selected host adapter consuming common runtime health snapshots. +pub(crate) enum HostIntegration { + None, + Systemd(SystemdIntegration), +} + +impl HostIntegration { + /// Selects an adapter from CLI mode and process environment. + /// + /// # Errors + /// + /// Explicit systemd mode fails when `NOTIFY_SOCKET` or the helper is + /// unavailable. Auto mode selects none without a notification socket and + /// preserves warn-and-continue helper behavior after systemd selection. + pub(crate) fn from_environment(mode: HostIntegrationMode) -> anyhow::Result { + Self::select( + mode, + std::env::var_os("NOTIFY_SOCKET").is_some(), + Arc::new(ProcessSystemdCommand), + ) + } + + fn select( + mode: HostIntegrationMode, + notify_socket_present: bool, + command: Arc, + ) -> anyhow::Result { + match mode { + HostIntegrationMode::None => Ok(Self::None), + HostIntegrationMode::Auto if !notify_socket_present => Ok(Self::None), + HostIntegrationMode::Auto => Ok(Self::Systemd(SystemdIntegration { + command, + strict: false, + ready_sent: false, + stopping_sent: false, + })), + HostIntegrationMode::Systemd => { + if !notify_socket_present { + anyhow::bail!( + "--host-integration systemd requires the NOTIFY_SOCKET environment variable" + ); + } + if !command.is_available() { + anyhow::bail!( + "--host-integration systemd requires an executable systemd-notify helper" + ); + } + Ok(Self::Systemd(SystemdIntegration { + command, + strict: true, + ready_sent: false, + stopping_sent: false, + })) + } + } + } + + /// Consumes snapshots until daemon cancellation or a strict notification failure. + pub(crate) async fn run( + mut self, + mut health: watch::Receiver, + cancellation: CancellationToken, + ) -> anyhow::Result<()> { + match &mut self { + Self::None => { + cancellation.cancelled().await; + Ok(()) + } + Self::Systemd(integration) => { + integration.publish(&health.borrow().clone())?; + loop { + tokio::select! { + biased; + result = health.changed() => { + if result.is_err() { + return Ok(()); + } + integration.publish(&health.borrow().clone())?; + } + () = cancellation.cancelled() => return Ok(()), + } + } + } + } + } +} + +pub(crate) struct SystemdIntegration { + command: Arc, + strict: bool, + ready_sent: bool, + stopping_sent: bool, +} + +impl SystemdIntegration { + fn publish(&mut self, snapshot: &RuntimeHealthSnapshot) -> anyhow::Result<()> { + let mut arguments = vec!["--pid=parent".to_owned()]; + arguments.push(format!("--status={}", status_text(snapshot))); + + if snapshot.is_readiness_serving() && !self.ready_sent { + arguments.push("--ready".to_owned()); + self.ready_sent = true; + } + if snapshot.lifecycle() == RuntimeLifecycle::Draining && !self.stopping_sent { + arguments.push("--stopping".to_owned()); + self.stopping_sent = true; + } + + if let Err(error) = self.command.execute(&arguments) { + if self.strict { + return Err(error); + } + log::warn!("Failed to publish automatic systemd notification"); + } + Ok(()) + } +} + +fn status_text(snapshot: &RuntimeHealthSnapshot) -> &'static str { + match snapshot.lifecycle() { + RuntimeLifecycle::Draining => "Stopping local TACACS+ services", + RuntimeLifecycle::Stopped => "Stopped", + RuntimeLifecycle::Failed => "Fatal runtime failure", + RuntimeLifecycle::Starting if !snapshot.has_applied_configuration() => { + "Waiting for initial validated configuration" + } + RuntimeLifecycle::Starting => "Waiting for enabled listeners", + RuntimeLifecycle::Serving if !snapshot.degradation_reasons().is_empty() => { + "Serving in degraded state" + } + RuntimeLifecycle::Serving if !snapshot.is_readiness_serving() => { + "Serving local endpoints without an eligible upstream server" + } + RuntimeLifecycle::Serving => "Ready to serve local TACACS+ requests", + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use tacacsrs_agent::{ + DegradationReason, EnabledServices, ListenerState, RuntimeHealthPublisher, RuntimeService, + }; + + use super::*; + + #[derive(Default)] + struct CapturingCommand { + available: bool, + fail: bool, + calls: Mutex>>, + } + + impl SystemdCommand for CapturingCommand { + fn is_available(&self) -> bool { + self.available + } + + fn execute(&self, arguments: &[String]) -> anyhow::Result<()> { + self.calls + .lock() + .expect("calls lock") + .push(arguments.to_vec()); + if self.fail { + anyhow::bail!("injected notification failure"); + } + Ok(()) + } + } + + fn ready_publisher() -> RuntimeHealthPublisher { + let health = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + health.set_applied_configuration(true); + health.set_eligible_server_count(1); + assert!(health.set_listener(RuntimeService::ClientApi, ListenerState::Bound)); + health + } + + #[test] + fn auto_without_socket_and_none_with_socket_select_no_integration() { + assert!(matches!( + HostIntegration::select( + HostIntegrationMode::Auto, + false, + Arc::new(CapturingCommand::default()), + ) + .expect("auto selection"), + HostIntegration::None, + )); + assert!(matches!( + HostIntegration::select( + HostIntegrationMode::None, + true, + Arc::new(CapturingCommand::default()), + ) + .expect("none selection"), + HostIntegration::None, + )); + } + + #[test] + fn strict_systemd_requires_socket_and_helper() { + let no_socket = HostIntegration::select( + HostIntegrationMode::Systemd, + false, + Arc::new(CapturingCommand { + available: true, + ..Default::default() + }), + ); + assert!(no_socket.is_err()); + + let no_helper = HostIntegration::select( + HostIntegrationMode::Systemd, + true, + Arc::new(CapturingCommand::default()), + ); + assert!(no_helper.is_err()); + } + + #[tokio::test] + async fn systemd_publishes_ready_once_degraded_status_and_stopping() { + let command = Arc::new(CapturingCommand { + available: true, + ..Default::default() + }); + let health = ready_publisher(); + let integration = HostIntegration::select( + HostIntegrationMode::Systemd, + true, + Arc::clone(&command) as Arc, + ) + .expect("systemd selection"); + let cancellation = CancellationToken::new(); + let child = cancellation.clone(); + let task = tokio::spawn(integration.run(health.subscribe(), child)); + tokio::task::yield_now().await; + + health.set_degraded(DegradationReason::UpstreamsUnavailable, true); + tokio::task::yield_now().await; + health.set_lifecycle(RuntimeLifecycle::Draining); + tokio::task::yield_now().await; + cancellation.cancel(); + task.await + .expect("task should join") + .expect("notifications should succeed"); + + let calls = command.calls.lock().expect("calls lock"); + assert_eq!( + calls + .iter() + .filter(|call| call.contains(&"--ready".to_owned())) + .count(), + 1, + ); + assert!(calls.iter().any(|call| call + .iter() + .any(|arg| arg == "--status=Serving in degraded state"))); + assert!(calls + .iter() + .any(|call| call.contains(&"--stopping".to_owned()))); + let command_text = format!("{calls:?}"); + for forbidden in [ + "redis://", + "unix://", + "192.0.2.10", + "credential-reference", + "test-secret", + ] { + assert!(!command_text.contains(forbidden), "systemd arguments exposed {forbidden}"); + } + } + + #[tokio::test] + async fn auto_ignores_notification_failure_but_strict_returns_it() { + let health = ready_publisher(); + let auto = HostIntegration::select( + HostIntegrationMode::Auto, + true, + Arc::new(CapturingCommand { + fail: true, + ..Default::default() + }), + ) + .expect("auto systemd selection"); + let cancellation = CancellationToken::new(); + cancellation.cancel(); + auto.run(health.subscribe(), cancellation) + .await + .expect("auto should tolerate failure"); + + let strict = HostIntegration::select( + HostIntegrationMode::Systemd, + true, + Arc::new(CapturingCommand { + available: true, + fail: true, + ..Default::default() + }), + ) + .expect("strict selection"); + assert!(strict + .run(health.subscribe(), CancellationToken::new()) + .await + .is_err()); + } +} diff --git a/executables/tacacsrs_agentd/src/main.rs b/executables/tacacsrs_agentd/src/main.rs index 615dfac4..d2012f20 100644 --- a/executables/tacacsrs_agentd/src/main.rs +++ b/executables/tacacsrs_agentd/src/main.rs @@ -7,26 +7,30 @@ use std::time::Duration; use anyhow::Context; use clap::Parser; -use futures_util::StreamExt; -use tacacsrs_agent::{EnabledServices, ServiceConfig, TacacsClientService}; +use tacacsrs_agent::{ + EnabledServices, ProxyDownstreamObfuscation, RuntimeHealthPublisher, ServiceConfig, + TacacsClientService, +}; use tacacsrs_agent_client::IpcEndpoint; use tacacsrs_cli_datastore::{ CliConfigSource, CliDatastoreInput, CliFileDatastore, CliSecurity, CliSecurityInputs, CliServerInput, }; use tacacsrs_cli_datastore::{CliPskInputs, PskKeyExchangeMode, PskKeyMaterial}; -use tacacsrs_config::TacacsPlusServerExt; -use tacacsrs_datastore::{ConfigChange, ConfigDatastore}; +use tacacsrs_datastore::{ConfigDatastore, InitialLoadPolicy}; use tacacsrs_sonic::{SonicConfigDb, SonicConnection, DEFAULT_REDIS_URL}; +use tokio_util::sync::CancellationToken; mod cli; mod config_filter; -mod systemd_notify; +mod config_supervisor; +mod host_integration; use crate::cli::{Cli, ServiceMode}; use crate::cli::PskKeyExchange; -use crate::config_filter::{config_filter_from_runtime_options, TacacsPlusFilter}; -use crate::systemd_notify::SystemdNotifier; +use crate::config_filter::{TacacsPlusFilter, config_filter_from_runtime_options}; +use crate::config_supervisor::ConfigSupervisor; +use crate::host_integration::HostIntegration; #[cfg(unix)] fn parse_socket_mode(mode: &str) -> anyhow::Result { @@ -164,65 +168,63 @@ fn build_datastore(cli: &Cli) -> Arc { Arc::new(CliFileDatastore::new(cli_datastore_input_from_cli(cli))) } -async fn apply_config_change( - label: &str, - change: ConfigChange, - service: &TacacsClientService, - config_filter: &dyn TacacsPlusFilter, -) -> anyhow::Result<()> { - log::info!( - "Datastore '{label}' reports configuration change: {} server(s); added={:?} removed={:?} modified={:?} root_metadata_changed={}", - change.config.server.len(), - change.delta.added_servers, - change.delta.removed_servers, - change.delta.modified_servers, - change.delta.root_metadata_changed, - ); - let filtered = config_filter.filter((*change.config).clone()).await?; - service - .reload_tacacs_plus_with_proxy_downstream_obfuscation( - filtered.tacacs_plus, - filtered.proxy_downstream_obfuscation, - ) - .await -} - -/// Spawn a background task that consumes [`ConfigDatastore::subscribe`] -/// events and applies each new snapshot to the running service. -fn spawn_change_listener( +async fn run_supervised_service( datastore: Arc, service: Arc, - status_notifier: Arc, + health: RuntimeHealthPublisher, config_filter: Arc, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - let label = datastore.label(); - let mut stream = match datastore.subscribe().await { - Ok(stream) => stream, - Err(error) => { - log::warn!("Datastore '{label}' does not support change notifications: {error:#}"); - return; + host_integration: HostIntegration, +) -> anyhow::Result<()> { + let datastore_policy = datastore.runtime_policy(); + let supervisor = ConfigSupervisor::new( + Arc::clone(&datastore), + Arc::clone(&service), + health.clone(), + config_filter, + ); + let cancellation = CancellationToken::new(); + + if datastore_policy.initial_load == InitialLoadPolicy::FailFast { + supervisor.load_initial(&cancellation).await?; + } + + let mut host_task = { + let cancellation = cancellation.clone(); + tokio::spawn(host_integration.run(health.subscribe(), cancellation)) + }; + + let supervisor_task = { + let cancellation = cancellation.clone(); + tokio::spawn(async move { + if datastore_policy.initial_load == InitialLoadPolicy::RetryUntilAvailable { + supervisor.run(&cancellation).await + } else { + supervisor.run_notifications(&cancellation).await; + Ok(()) } - }; - log::info!("Subscribed to '{label}' configuration change notifications"); - while let Some(change) = stream.next().await { - match apply_config_change(label, change, &service, config_filter.as_ref()).await { - Ok(()) => { - log::info!( - "Applied datastore '{label}' configuration reload with {} upstream server(s) supporting authentication, authorization, and accounting", - service.server_count(), - ); - status_notifier.publish_server_state(service.server_count()); - } - Err(error) => { - log::error!( - "Failed to apply datastore '{label}' configuration reload; keeping previous runtime state: {error:#}" - ); - } + }) + }; + + let mut host_task_completed = false; + let service_result = tokio::select! { + result = service.serve() => result, + result = &mut host_task => { + host_task_completed = true; + match result.context("Host integration task failed")? { + Ok(()) => Err(anyhow::anyhow!("Host integration stopped unexpectedly")), + Err(error) => Err(error.context("Host integration failed")), } } - log::debug!("Datastore '{label}' change stream ended"); - }) + }; + cancellation.cancel(); + supervisor_task + .await + .context("Configuration supervisor task failed")??; + if !host_task_completed { + host_task.await.context("Host integration task failed")??; + } + + service_result } /// Starts the central TACACS+ client service process. @@ -234,6 +236,7 @@ fn spawn_change_listener( async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); init_logger(cli.verbose); + let host_integration = HostIntegration::from_environment(cli.host_integration)?; let enabled_services = enabled_services_from_cli(&cli); let endpoint = cli @@ -283,54 +286,26 @@ async fn main() -> anyhow::Result<()> { ); let datastore = build_datastore(&cli); - let tacacs_plus = { - let initial = datastore.load().await.with_context(|| { - format!("Failed to load configuration from datastore '{}'", datastore.label()) - })?; - log::info!("Initial configuration loaded from datastore '{}'", datastore.label()); - config_filter.filter(initial).await? - }; - let proxy_downstream_obfuscation = tacacs_plus.proxy_downstream_obfuscation; - let tacacs_plus = tacacs_plus.tacacs_plus; - - log::info!( - "Upstream servers: {} configured, probe interval: {}s", - tacacs_plus.server.len(), - cli.preferred_probe_interval_seconds, - ); - for server in &tacacs_plus.server { - let security_label = if server.is_tls() { - "TLS" - } else { - "obfuscation" - }; - log::info!(" {} ({}) -> {}:{}", server.name, security_label, server.address, server.port); - } - + let health = RuntimeHealthPublisher::new(enabled_services); + let empty_tacacs_plus = tacacsrs_config::TacacsPlus::empty(); let service = Arc::new( - TacacsClientService::new(ServiceConfig { - enabled_services, - endpoint, - proxy_endpoint, - proxy_downstream_obfuscation, - tacacs_plus, - preferred_probe_interval: Duration::from_secs(cli.preferred_probe_interval_seconds), - #[cfg(unix)] - socket_mode: parse_socket_mode(&cli.socket_mode)?, - disable_certificate_verification: cli.insecure_disable_certificate_verification, - }) + TacacsClientService::waiting_for_configuration( + ServiceConfig { + enabled_services, + endpoint, + proxy_endpoint, + proxy_downstream_obfuscation: ProxyDownstreamObfuscation::default(), + tacacs_plus: empty_tacacs_plus, + preferred_probe_interval: Duration::from_secs(cli.preferred_probe_interval_seconds), + #[cfg(unix)] + socket_mode: parse_socket_mode(&cli.socket_mode)?, + disable_certificate_verification: cli.insecure_disable_certificate_verification, + }, + health.clone(), + ) .context("Failed to build TACACS+ client service configuration")?, ); - let status_notifier = Arc::new(SystemdNotifier::from_env()); - status_notifier.publish_server_state(service.server_count()); - - let _change_listener = spawn_change_listener( - Arc::clone(&datastore), - Arc::clone(&service), - Arc::clone(&status_notifier), - Arc::clone(&config_filter), - ); - service.serve().await + run_supervised_service(datastore, service, health, config_filter, host_integration).await } #[cfg(test)] @@ -338,24 +313,15 @@ mod tests { use clap::Parser; use std::fs; use std::path::PathBuf; - use std::sync::Arc; - use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; - use super::{Cli, apply_config_change, cli_datastore_input_from_cli, enabled_services_from_cli}; - use crate::config_filter::{NoopTacacsPlusFilter, ProxySelfLoopFilter}; - use tacacsrs_agent::{ - EnabledServices, ProxyDownstreamObfuscation, ServiceConfig, TacacsClientService, - }; - use tacacsrs_agent_client::IpcEndpoint; + use super::{Cli, cli_datastore_input_from_cli, enabled_services_from_cli}; + use crate::cli::HostIntegrationMode; + use tacacsrs_agent::EnabledServices; use tacacsrs_config::PskDheKeSupportedGroup; use tacacsrs_config::crypto_types::PrivateKeyFormat; - use tacacsrs_config::{ - TacacsPlus, TacacsPlusBuilder, TacacsPlusServerBuilder, TacacsPlusServerType, - ValidationOptions, - }; + use tacacsrs_config::ValidationOptions; use tacacsrs_cli_datastore::{tacacs_plus_from_cli_input, tacacs_plus_from_file}; - use tacacsrs_datastore::{ConfigChange, ConfigDelta}; fn sample_path(file_name: &str) -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -378,42 +344,6 @@ mod tests { path } - fn test_config(addresses: &[&str]) -> TacacsPlus { - addresses - .iter() - .enumerate() - .map(|(index, address)| { - let (host, port) = address.rsplit_once(':').unwrap_or((*address, "49")); - TacacsPlusServerBuilder::new( - format!("server-{index}"), - TacacsPlusServerType::AUTHENTICATION - | TacacsPlusServerType::AUTHORIZATION - | TacacsPlusServerType::ACCOUNTING, - host.to_owned(), - port.parse().expect("test port should be valid"), - ) - .with_shared_secret("test-secret".to_owned()) - }) - .fold(TacacsPlusBuilder::new(), TacacsPlusBuilder::with_server_builder) - .build() - .expect("test config should be valid") - } - - fn test_service(config: TacacsPlus) -> TacacsClientService { - TacacsClientService::new(ServiceConfig { - enabled_services: EnabledServices::CLIENT_API, - endpoint: IpcEndpoint::default_local(), - proxy_endpoint: None, - proxy_downstream_obfuscation: ProxyDownstreamObfuscation::default(), - tacacs_plus: config, - preferred_probe_interval: Duration::from_secs(1), - #[cfg(unix)] - socket_mode: 0o660, - disable_certificate_verification: false, - }) - .expect("test service should be valid") - } - #[test] fn tacacs_plus_from_config_loads_all_servers() { let path = write_temp_config( @@ -506,6 +436,26 @@ mod tests { ]); assert_eq!(enabled_services_from_cli(&cli), EnabledServices::CLIENT_API); + assert_eq!(cli.host_integration, HostIntegrationMode::Auto); + } + + #[test] + fn host_integration_accepts_explicit_none_and_systemd() { + for (value, expected) in [ + ("none", HostIntegrationMode::None), + ("systemd", HostIntegrationMode::Systemd), + ] { + let cli = Cli::parse_from([ + "tacacsrs-agentd", + "--server-addr", + "192.0.2.20:49", + "--shared-secret", + "secret1", + "--host-integration", + value, + ]); + assert_eq!(cli.host_integration, expected); + } } #[test] @@ -567,46 +517,6 @@ mod tests { assert!(result.is_err()); } - #[tokio::test] - async fn apply_config_change_reloads_service_without_restart() { - let initial = test_config(&["192.0.2.10:49"]); - let service = test_service(initial.clone()); - assert_eq!(service.server_count(), 1); - - let updated = test_config(&["192.0.2.10:49", "192.0.2.11:49"]); - let change = ConfigChange { - delta: ConfigDelta::diff(Some(&initial), &updated), - config: Arc::new(updated), - }; - - apply_config_change("test", change, &service, &NoopTacacsPlusFilter::default()) - .await - .unwrap(); - - assert_eq!(service.server_count(), 2); - } - - #[tokio::test] - async fn apply_config_change_filters_proxy_self_loop_on_reload() { - let initial = test_config(&["192.0.2.10:49"]); - let service = test_service(initial.clone()); - assert_eq!(service.server_count(), 1); - - let updated = test_config(&["127.0.0.1:9050", "192.0.2.11:49"]); - let change = ConfigChange { - delta: ConfigDelta::diff(Some(&initial), &updated), - config: Arc::new(updated), - }; - let filter = - ProxySelfLoopFilter::new("127.0.0.1:9050".parse().expect("socket should parse")); - - apply_config_change("test", change, &service, &filter) - .await - .unwrap(); - - assert_eq!(service.server_count(), 1); - } - #[test] fn tacacs_plus_from_cli_accepts_pem_client_identity_files() { let cert_path = sample_path("client.crt"); diff --git a/executables/tacacsrs_agentd/src/systemd_notify.rs b/executables/tacacsrs_agentd/src/systemd_notify.rs deleted file mode 100644 index 8bb7f532..00000000 --- a/executables/tacacsrs_agentd/src/systemd_notify.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! Minimal systemd status notifications for the agent daemon. - -use std::process::Command; -use std::sync::atomic::{AtomicBool, Ordering}; - -/// Thin wrapper around the `systemd-notify` helper. -/// -/// The daemon only uses this when `NOTIFY_SOCKET` is present, which means it -/// is running under a `Type=notify` unit. Initial empty-config startup sends a -/// status line without `READY=1`, keeping the unit in `activating` until a -/// TACACS+ server appears in the datastore. -#[derive(Debug, Default)] -pub(crate) struct SystemdNotifier { - enabled: bool, - ready_sent: AtomicBool, -} - -impl SystemdNotifier { - pub(crate) fn from_env() -> Self { - Self { - enabled: std::env::var_os("NOTIFY_SOCKET").is_some(), - ready_sent: AtomicBool::new(false), - } - } - - pub(crate) fn publish_server_state(&self, server_count: usize) { - if !self.enabled { - return; - } - - if server_count == 0 { - Self::notify( - false, - "Waiting for TACACS+ servers that support authentication, authorization, and accounting from SONiC ConfigDB", - ); - return; - } - - let ready = !self.ready_sent.swap(true, Ordering::AcqRel); - let status = format!( - "Serving with {server_count} TACACS+ upstream server(s) supporting authentication, authorization, and accounting" - ); - Self::notify(ready, &status); - } - - fn notify(ready: bool, status: &str) { - let mut command = Command::new("systemd-notify"); - command.arg("--pid=parent"); - command.arg(format!("--status={status}")); - if ready { - command.arg("--ready"); - } - - match command.status() { - Ok(exit_status) if exit_status.success() => {} - Ok(exit_status) => { - log::warn!( - "systemd-notify exited with status {exit_status} while publishing status '{status}'" - ); - } - Err(error) => { - log::warn!( - "Failed to execute systemd-notify while publishing status '{status}': {error}" - ); - } - } - } -} diff --git a/executables/tacacsrs_agentd/tests/health_probe.rs b/executables/tacacsrs_agentd/tests/health_probe.rs new file mode 100644 index 00000000..84291331 --- /dev/null +++ b/executables/tacacsrs_agentd/tests/health_probe.rs @@ -0,0 +1,155 @@ +use std::net::SocketAddr; +use std::process::Output; + +use tacacsrs_agent_client::health::{HealthClient, READINESS_HEALTH_SERVICE}; +use tacacsrs_agent_client::IpcEndpoint; +use tokio::net::TcpListener; +use tokio::process::Command; +use tokio_stream::wrappers::TcpListenerStream; +use tokio_util::sync::CancellationToken; +use tonic::Code; +use tonic_health::ServingStatus; + +const EXIT_NOT_SERVING: i32 = 1; +const EXIT_CHECK_ERROR: i32 = 3; + +async fn run_probe(endpoint: &str, check: &str, timeout_seconds: u64) -> Output { + Command::new(env!("CARGO_BIN_EXE_tacacsrs-agent-health")) + .args([ + "--endpoint", + endpoint, + "--check", + check, + "--timeout-seconds", + &timeout_seconds.to_string(), + ]) + .output() + .await + .expect("probe should run") +} + +async fn start_tcp_health( + status: ServingStatus, +) -> (SocketAddr, CancellationToken, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("health listener should bind"); + let address = listener.local_addr().expect("listener address"); + let incoming = TcpListenerStream::new(listener); + let (reporter, service) = tonic_health::server::health_reporter(); + reporter + .set_service_status(READINESS_HEALTH_SERVICE, status) + .await; + let cancellation = CancellationToken::new(); + let child = cancellation.clone(); + let task = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming_shutdown(incoming, child.cancelled_owned()) + .await + .expect("health server should run"); + }); + (address, cancellation, task) +} + +#[tokio::test] +async fn tcp_probe_exit_codes_are_stable() { + let (address, cancellation, task) = start_tcp_health(ServingStatus::Serving).await; + let endpoint = address.to_string(); + + let serving = run_probe(&endpoint, "readiness", 2).await; + assert!(serving.status.success()); + + cancellation.cancel(); + task.await.expect("server task should join"); + let unavailable = run_probe(&endpoint, "readiness", 2).await; + assert_eq!(unavailable.status.code(), Some(EXIT_CHECK_ERROR)); + let diagnostic = String::from_utf8_lossy(&unavailable.stderr); + for forbidden in [ + "redis://", + "192.0.2.10", + "credential-reference", + "test-secret", + ] { + assert!(!diagnostic.contains(forbidden), "probe output exposed {forbidden}"); + } +} + +#[tokio::test] +async fn not_serving_and_unknown_service_are_distinct_standard_results() { + let (address, cancellation, task) = start_tcp_health(ServingStatus::NotServing).await; + let endpoint = IpcEndpoint::Tcp(address); + + let not_serving = run_probe(&address.to_string(), "readiness", 2).await; + assert_eq!(not_serving.status.code(), Some(EXIT_NOT_SERVING)); + + let mut client = HealthClient::connect(&endpoint) + .await + .expect("health client should connect"); + let error = client + .check("tacacsrs.agent.health.v1.Unknown") + .await + .expect_err("unknown service should fail"); + assert_eq!(error.code(), Code::NotFound); + + cancellation.cancel(); + task.await.expect("server task should join"); +} + +#[tokio::test] +async fn timeout_returns_check_error() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("silent listener should bind"); + let endpoint = listener.local_addr().expect("listener address").to_string(); + let connection = tokio::spawn(async move { + let (_stream, _) = listener.accept().await.expect("probe should connect"); + std::future::pending::<()>().await; + }); + + let output = run_probe(&endpoint, "liveness", 1).await; + + connection.abort(); + assert_eq!(output.status.code(), Some(EXIT_CHECK_ERROR)); + assert!(String::from_utf8_lossy(&output.stderr).contains("timed out")); +} + +#[cfg(unix)] +#[tokio::test] +async fn unix_socket_probe_reports_serving() { + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + use tokio::net::UnixListener; + use tokio_stream::wrappers::UnixListenerStream; + + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be valid") + .as_nanos(); + let path = PathBuf::from(format!("/tmp/tacacsrs-health-probe-{unique}.sock")); + let listener = UnixListener::bind(&path).expect("health socket should bind"); + let incoming = UnixListenerStream::new(listener); + let (reporter, service) = tonic_health::server::health_reporter(); + reporter + .set_service_status(READINESS_HEALTH_SERVICE, ServingStatus::Serving) + .await; + let cancellation = CancellationToken::new(); + let child = cancellation.clone(); + let task = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming_shutdown(incoming, child.cancelled_owned()) + .await + .expect("health server should run"); + }); + + let output = run_probe(path.to_str().expect("UTF-8 path"), "readiness", 2).await; + + cancellation.cancel(); + task.await.expect("server task should join"); + tokio::fs::remove_file(path) + .await + .expect("remove health socket"); + assert!(output.status.success()); +} diff --git a/lde/run-p1-health-process-smoke.sh b/lde/run-p1-health-process-smoke.sh new file mode 100644 index 00000000..934d22f9 --- /dev/null +++ b/lde/run-p1-health-process-smoke.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +if [[ -f "$HOME/.cargo/env" ]]; then + # shellcheck source=/dev/null + source "$HOME/.cargo/env" +fi + +container_runtime="${CONTAINER_RUNTIME:-podman.exe}" +redis_image="${REDIS_IMAGE:-docker.io/library/redis:7-alpine}" +redis_name="tacacsrs-p1-process-redis" +redis_port="${REDIS_PORT:-6387}" +proxy_port="${PROXY_PORT:-10491}" +socket_path="/tmp/tacacsrs-p1-process.sock" +strict_socket_path="/tmp/tacacsrs-p1-strict.sock" +auto_socket_path="/tmp/tacacsrs-p1-auto.sock" +agent_pid="" + +cleanup() { + if [[ -n "$agent_pid" ]]; then + kill -TERM "$agent_pid" >/dev/null 2>&1 || true + fi + "$container_runtime" rm -f "$redis_name" >/dev/null 2>&1 || true + rm -f "$socket_path" "$strict_socket_path" "$auto_socket_path" +} +trap cleanup EXIT + +export CARGO_INCREMENTAL=0 +cargo build -p tacacsrs-agentd --bins >/dev/null +agent="target/debug/tacacsrs-agentd" +probe="target/debug/tacacsrs-agent-health" + +rm -f "$socket_path" "$strict_socket_path" "$auto_socket_path" +"$container_runtime" rm -f "$redis_name" >/dev/null 2>&1 || true +python3 -c 'import socket; s = socket.socket(socket.AF_UNIX); s.bind("/tmp/tacacsrs-p1-process.sock"); s.close()' + +NOTIFY_SOCKET=/tmp/nonexistent-notify "$agent" \ + --sonic \ + --sonic-redis-url "redis://127.0.0.1:${redis_port}" \ + --sonic-redis-db 4 \ + --service-mode both \ + --listen-endpoint "$socket_path" \ + --proxy-endpoint "127.0.0.1:${proxy_port}" \ + --host-integration none \ + -vv >/dev/null 2>&1 & +agent_pid=$! + +live=0 +for _ in $(seq 1 80); do + kill -0 "$agent_pid" + if "$probe" --endpoint "$socket_path" --check liveness --timeout-seconds 2 >/dev/null 2>&1; then + live=1 + break + fi +done +[[ "$live" -eq 1 ]] + +if "$probe" --endpoint "$socket_path" --check startup --timeout-seconds 2 >/dev/null 2>&1; then + echo "startup unexpectedly serving before Redis" >&2 + exit 31 +else + exit_code=$? + [[ "$exit_code" -eq 1 ]] +fi +"$probe" --endpoint "$socket_path" --check liveness --timeout-seconds 2 >/dev/null +if "$probe" --endpoint "$socket_path" --check readiness --timeout-seconds 2 >/dev/null 2>&1; then + echo "readiness unexpectedly serving before Redis" >&2 + exit 32 +else + exit_code=$? + [[ "$exit_code" -eq 1 ]] +fi + +"$container_runtime" run -d \ + --name "$redis_name" \ + -p "${redis_port}:6379" \ + "$redis_image" >/dev/null + +for _ in $(seq 1 30); do + if "$container_runtime" exec "$redis_name" redis-cli PING 2>/dev/null | grep -q PONG; then + break + fi +done +"$container_runtime" exec "$redis_name" redis-cli PING | grep -q PONG +"$container_runtime" exec "$redis_name" redis-cli -n 4 \ + CONFIG SET notify-keyspace-events KEA >/dev/null +"$container_runtime" exec "$redis_name" redis-cli -n 4 \ + HSET 'TACPLUS|global' timeout 5 auth_type pap >/dev/null +"$container_runtime" exec "$redis_name" redis-cli -n 4 \ + HSET 'TACPLUS_SERVER|192.0.2.10' priority 64 tcp_port 49 timeout 5 >/dev/null + +ready=0 +for _ in $(seq 1 80); do + if "$probe" --endpoint "$socket_path" --check readiness --timeout-seconds 2 >/dev/null 2>&1; then + ready=1 + break + fi +done +[[ "$ready" -eq 1 ]] + +exec 9<>"/dev/tcp/127.0.0.1/${proxy_port}" +printf '\300' >&9 + +kill -TERM "$agent_pid" +kill -0 "$agent_pid" +exec 9>&- +wait "$agent_pid" +agent_pid="" +[[ ! -e "$socket_path" ]] +if bash -c "exec 9<>/dev/tcp/127.0.0.1/${proxy_port}" 2>/dev/null; then + echo "proxy port remained bound after SIGTERM" >&2 + exit 33 +fi + +if env -u NOTIFY_SOCKET "$agent" \ + --server-addr 192.0.2.10:49 \ + --shared-secret placeholder \ + --host-integration systemd \ + --listen-endpoint "$strict_socket_path" >/dev/null 2>&1; then + echo "strict systemd mode started without NOTIFY_SOCKET" >&2 + exit 34 +fi + +NOTIFY_SOCKET=/tmp/nonexistent-notify "$agent" \ + --server-addr 192.0.2.10:49 \ + --shared-secret placeholder \ + --host-integration auto \ + --listen-endpoint "$auto_socket_path" >/dev/null 2>&1 & +agent_pid=$! +auto_live=0 +for _ in $(seq 1 80); do + kill -0 "$agent_pid" + if "$probe" --endpoint "$auto_socket_path" --check liveness --timeout-seconds 2 >/dev/null 2>&1; then + auto_live=1 + break + fi +done +[[ "$auto_live" -eq 1 ]] +kill -TERM "$agent_pid" +wait "$agent_pid" +agent_pid="" +[[ ! -e "$auto_socket_path" ]] + +echo "PASS: stale UDS replacement, pre-Redis startup, health recovery, both listeners, active drain, none/auto modes, SIGTERM cleanup, port release, and strict systemd prerequisite failure." \ No newline at end of file diff --git a/libraries/tacacsrs_agent/Cargo.toml b/libraries/tacacsrs_agent/Cargo.toml index cb106da3..b4fd5147 100644 --- a/libraries/tacacsrs_agent/Cargo.toml +++ b/libraries/tacacsrs_agent/Cargo.toml @@ -21,6 +21,10 @@ tacacsrs-networking = { path = "../tacacsrs_networking" } tokio = { version = "1.52.3", features = ["full"] } tokio-stream = { version = "0.1.18", features = ["net"] } tonic = "0.14.6" +tonic-health = "0.14.6" + +[dev-dependencies] +tokio = { version = "1.52.3", features = ["test-util"] } [lints] workspace = true diff --git a/libraries/tacacsrs_agent/README.md b/libraries/tacacsrs_agent/README.md index de4b5892..a35755c4 100644 --- a/libraries/tacacsrs_agent/README.md +++ b/libraries/tacacsrs_agent/README.md @@ -221,8 +221,9 @@ On Unix systems the listener follows this startup sequence: 6. begin serving IPC clients ``` -Shutdown stops accepting new IPC connections first, waits for active RPCs to -finish, and then removes the Unix socket path. +One coordinator receives SIGTERM or Ctrl-C, changes the shared lifecycle to `Draining`, and then broadcasts shutdown to every listener. Registration guards publish `Binding`, `Bound`, and `Stopped` on every return path. Shutdown stops accepting new local work, waits for active gRPC and raw proxy requests to finish, removes Unix socket paths, and finally publishes `Stopped`. A listener bind or accept-loop failure marks fatal health, cancels siblings, and waits for their guards and cleanup before returning. + +The same typed watch-backed snapshot drives startup, liveness, readiness, datastore freshness, upstream availability, standard gRPC health, host integration, and operator status. It contains typed flags and counts only; configuration values, server identities, credential references, and raw errors stay outside the shared state. ## Configuration details diff --git a/libraries/tacacsrs_agent/src/lib.rs b/libraries/tacacsrs_agent/src/lib.rs index 5db7212b..ddcfb091 100644 --- a/libraries/tacacsrs_agent/src/lib.rs +++ b/libraries/tacacsrs_agent/src/lib.rs @@ -21,4 +21,8 @@ pub mod upstream; mod test_support; pub use config::{EnabledServices, ProxyDownstreamObfuscation, ServiceConfig}; -pub use runtime::TacacsClientService; +pub use runtime::{ + DatastoreState, DegradationReason, ListenerState, RuntimeHealthPublisher, + RuntimeHealthSnapshot, RuntimeLifecycle, RuntimeService, TacacsClientService, + UpstreamAvailability, +}; diff --git a/libraries/tacacsrs_agent/src/runtime/client_service.rs b/libraries/tacacsrs_agent/src/runtime/client_service.rs index 8b5332e9..4efd038c 100644 --- a/libraries/tacacsrs_agent/src/runtime/client_service.rs +++ b/libraries/tacacsrs_agent/src/runtime/client_service.rs @@ -26,7 +26,10 @@ use tacacsrs_config::TacacsPlus; use tokio::sync::RwLock; use tokio::task::JoinSet; -use super::{RequestTracker, enumerate_supported_servers}; +use super::{ + RequestTracker, RuntimeHealthPublisher, ShutdownCoordinator, UpstreamAvailability, + enumerate_supported_servers, +}; use crate::config::{ProxyDownstreamObfuscation, ServiceConfig}; use crate::services::client_api::ClientApiService; use crate::services::tacacs_proxy::TacacsProxyService; @@ -66,6 +69,8 @@ pub struct TacacsClientService { proxy_downstream_obfuscation: Arc>, /// Shared request lifecycle tracker used for graceful shutdown draining. request_tracker: Arc, + /// Common protocol-neutral runtime health publisher. + health: RuntimeHealthPublisher, } impl TacacsClientService { @@ -75,28 +80,72 @@ impl TacacsClientService { /// # Errors /// /// Returns an error if credential-reference resolution fails. - pub fn new(config: ServiceConfig) -> anyhow::Result { + pub fn new(config: ServiceConfig, health: RuntimeHealthPublisher) -> anyhow::Result { + Self::build(config, health, true) + } + + /// Builds a service that can bind local listeners while waiting for its + /// first valid datastore snapshot. + /// + /// The placeholder configuration must contain no upstream servers. A + /// successful [`reload_tacacs_plus_with_proxy_downstream_obfuscation`](Self::reload_tacacs_plus_with_proxy_downstream_obfuscation) + /// transition marks the first configuration as applied. + /// + /// # Errors + /// + /// Returns an error if the placeholder contains an upstream server, the + /// endpoint configuration is invalid, or the health publisher was created + /// for a different service selection. + pub fn waiting_for_configuration( + config: ServiceConfig, + health: RuntimeHealthPublisher, + ) -> anyhow::Result { + if !config.tacacs_plus.server.is_empty() { + anyhow::bail!("A service waiting for configuration requires an empty server set"); + } + + Self::build(config, health, false) + } + + fn build( + config: ServiceConfig, + health: RuntimeHealthPublisher, + configuration_applied: bool, + ) -> anyhow::Result { #[cfg(unix)] if config.enabled_services.client_api() { ClientApiService::validate_endpoint(&config.endpoint)?; } validate_enabled_services(&config)?; + if health.snapshot().enabled_services() != config.enabled_services { + anyhow::bail!( + "Runtime health and service configuration must enable the same local services" + ); + } let servers = enumerate_supported_servers(&config)?; + let eligible_server_count = servers.len(); let connector: Arc = Arc::new(NetworkUpstreamConnector { disable_certificate_verification: config.disable_certificate_verification, }); - let state = - Arc::new(UpstreamManager::new(servers, connector, config.preferred_probe_interval)); + let state = Arc::new(UpstreamManager::new( + servers, + connector, + config.preferred_probe_interval, + health.clone(), + )); let proxy_downstream_obfuscation = Arc::new(RwLock::new(config.proxy_downstream_obfuscation.clone())); let request_tracker = Arc::new(RequestTracker::default()); + health.set_eligible_server_count(eligible_server_count); + health.set_applied_configuration(configuration_applied); Ok(Self { config, state, proxy_downstream_obfuscation, request_tracker, + health, }) } @@ -104,24 +153,38 @@ impl TacacsClientService { pub(super) fn new_with_connector( config: ServiceConfig, connector: Arc, + health: RuntimeHealthPublisher, ) -> anyhow::Result { if config.enabled_services.client_api() { ClientApiService::validate_endpoint(&config.endpoint)?; } validate_enabled_services(&config)?; + if health.snapshot().enabled_services() != config.enabled_services { + anyhow::bail!( + "Runtime health and service configuration must enable the same local services" + ); + } let servers = enumerate_supported_servers(&config)?; + let eligible_server_count = servers.len(); - let state = - Arc::new(UpstreamManager::new(servers, connector, config.preferred_probe_interval)); + let state = Arc::new(UpstreamManager::new( + servers, + connector, + config.preferred_probe_interval, + health.clone(), + )); let proxy_downstream_obfuscation = Arc::new(RwLock::new(config.proxy_downstream_obfuscation.clone())); let request_tracker = Arc::new(RequestTracker::default()); + health.set_eligible_server_count(eligible_server_count); + health.set_applied_configuration(true); Ok(Self { config, state, proxy_downstream_obfuscation, request_tracker, + health, }) } @@ -155,8 +218,13 @@ impl TacacsClientService { reload_config.tacacs_plus = tacacs_plus; reload_config.proxy_downstream_obfuscation = proxy_downstream_obfuscation.clone(); let servers = enumerate_supported_servers(&reload_config)?; + let eligible_server_count = servers.len(); self.state.reload_servers(servers).await?; *self.proxy_downstream_obfuscation.write().await = proxy_downstream_obfuscation; + self.health.set_eligible_server_count(eligible_server_count); + self.health.set_applied_configuration(true); + self.health + .set_upstream_availability(UpstreamAvailability::Unknown); Ok(()) } @@ -180,29 +248,33 @@ impl TacacsClientService { /// Returns an error if the IPC listener cannot be created or if the local /// endpoint configuration is invalid for the current platform. pub async fn serve(&self) -> anyhow::Result<()> { + let shutdown = ShutdownCoordinator::new(self.health.clone()); + let signal_monitor = shutdown.spawn_process_signal_monitor(); + let result = self.serve_with_shutdown(&shutdown).await; + signal_monitor.abort(); + result + } + + async fn serve_with_shutdown(&self, shutdown: &ShutdownCoordinator) -> anyhow::Result<()> { log::info!("Warming upstream TACACS+ connections"); self.state.warm_connections().await; - let probe_task = if self.state.server_count() > 1 { - log::info!( - "Starting preferred-server probe task (interval: {:?})", - self.config.preferred_probe_interval, - ); - Some(self.state.spawn_preferred_probe()) - } else { - None - }; - - let result = self.serve_enabled_services().await; + log::info!( + "Starting dynamic preferred-server probe task (interval: {:?})", + self.config.preferred_probe_interval, + ); + let probe_task = self.state.spawn_preferred_probe(); - if let Some(task) = probe_task { - task.abort(); + let result = self.serve_enabled_services(shutdown).await; + probe_task.abort(); + if result.is_ok() { + shutdown.mark_stopped(); } log::info!("TACACS+ client service has shut down"); result } - async fn serve_enabled_services(&self) -> anyhow::Result<()> { + async fn serve_enabled_services(&self, shutdown: &ShutdownCoordinator) -> anyhow::Result<()> { let listener_options = ListenerOptions::from_config(&self.config); let mut tasks = JoinSet::new(); let mut service_count = 0; @@ -211,7 +283,13 @@ impl TacacsClientService { let service = ClientApiService::new(Arc::clone(&self.state), Arc::clone(&self.request_tracker)); let endpoint = self.config.endpoint.clone(); - tasks.spawn(async move { service.serve(&endpoint, listener_options).await }); + let shutdown = shutdown.subscribe(); + let health = self.health.clone(); + tasks.spawn(async move { + service + .serve(&endpoint, listener_options, shutdown, health) + .await + }); service_count += 1; } @@ -222,7 +300,13 @@ impl TacacsClientService { Arc::clone(&self.request_tracker), ); let endpoint = proxy_endpoint(&self.config)?.clone(); - tasks.spawn(async move { service.serve(&endpoint, listener_options).await }); + let shutdown = shutdown.subscribe(); + let health = self.health.clone(); + tasks.spawn(async move { + service + .serve(&endpoint, listener_options, shutdown, health) + .await + }); service_count += 1; } @@ -234,11 +318,13 @@ impl TacacsClientService { match result { Ok(Ok(())) => {} Ok(Err(error)) => { - tasks.abort_all(); + shutdown.fail(); + tasks.shutdown().await; return Err(error); } Err(error) => { - tasks.abort_all(); + shutdown.fail(); + tasks.shutdown().await; return Err(anyhow::anyhow!("Runtime service task failed: {error}")); } } @@ -287,8 +373,10 @@ mod tests { use crate::config::{EnabledServices, ProxyDownstreamObfuscation}; use super::TacacsClientService; - use crate::runtime::RequestTracker; - use crate::runtime::REQUIRED_SERVER_TYPES; + use crate::runtime::{ + ListenerState, REQUIRED_SERVER_TYPES, RequestTracker, RuntimeHealthPublisher, + RuntimeLifecycle, RuntimeService, ShutdownCoordinator, + }; use crate::config::ServiceConfig; use crate::services::client_api::GrpcService; use crate::test_support::{FakeConnection, FakeConnector, build_request}; @@ -341,6 +429,19 @@ mod tests { } } + fn test_service(config: ServiceConfig) -> anyhow::Result { + let health = RuntimeHealthPublisher::new(config.enabled_services); + TacacsClientService::new(config, health) + } + + fn test_service_with_connector( + config: ServiceConfig, + connector: Arc, + ) -> anyhow::Result { + let health = RuntimeHealthPublisher::new(config.enabled_services); + TacacsClientService::new_with_connector(config, connector, health) + } + #[test] fn service_uses_only_servers_supporting_runtime_operations() { let endpoint = test_endpoint("tacacs-service-runtime-filter"); @@ -351,7 +452,7 @@ mod tests { let config = service_config(endpoint, vec![partial, full]); let connector = Arc::new(FakeConnector::new(HashMap::new())); - let service = TacacsClientService::new_with_connector(config, connector).unwrap(); + let service = test_service_with_connector(config, connector).unwrap(); assert_eq!(service.state.server_count(), 1); } @@ -365,7 +466,7 @@ mod tests { let config = service_config(endpoint, vec![partial]); let connector = Arc::new(FakeConnector::new(HashMap::new())); - let service = TacacsClientService::new_with_connector(config, connector) + let service = test_service_with_connector(config, connector) .expect("service should accept waiting-for-config state"); assert_eq!(service.state.server_count(), 0); @@ -376,7 +477,7 @@ mod tests { let endpoint = IpcEndpoint::Tcp("127.0.0.1:0".parse().expect("test endpoint is valid")); let config = service_config(endpoint, vec![test_server("primary:49")]); - let Err(error) = TacacsClientService::new(config) else { + let Err(error) = test_service(config) else { panic!("service should reject TCP client API endpoints on Unix"); }; @@ -393,8 +494,8 @@ mod tests { config.proxy_endpoint = Some(IpcEndpoint::Tcp("127.0.0.1:0".parse().expect("test endpoint is valid"))); - let service = TacacsClientService::new(config) - .expect("service should accept TCP proxy endpoints on Unix"); + let service = + test_service(config).expect("service should accept TCP proxy endpoints on Unix"); assert_eq!(service.state.server_count(), 1); } @@ -407,7 +508,7 @@ mod tests { config.proxy_endpoint = Some(IpcEndpoint::Tcp("127.0.0.1:1".parse().expect("test endpoint is valid"))); - let service = TacacsClientService::new(config) + let service = test_service(config) .expect("proxy-only mode should not validate the disabled client API endpoint"); assert_eq!(service.state.server_count(), 1); @@ -419,7 +520,7 @@ mod tests { let mut config = service_config(endpoint, vec![test_server("primary:49")]); config.enabled_services = EnabledServices::NONE; - let Err(error) = TacacsClientService::new(config) else { + let Err(error) = test_service(config) else { panic!("service should reject configurations with no enabled services"); }; @@ -434,7 +535,7 @@ mod tests { let mut config = service_config(endpoint, vec![test_server("primary:49")]); config.enabled_services = EnabledServices::TACACS_PROXY; - let Err(error) = TacacsClientService::new(config) else { + let Err(error) = test_service(config) else { panic!("service should reject proxy mode without a proxy endpoint"); }; @@ -481,7 +582,7 @@ mod tests { vec![test_server("primary:49"), test_server("secondary:49")], ); - let service = TacacsClientService::new_with_connector(config, connector).unwrap(); + let service = test_service_with_connector(config, connector).unwrap(); let service_task = tokio::spawn(async move { service.serve().await }); tokio::time::sleep(Duration::from_millis(50)).await; @@ -533,7 +634,7 @@ mod tests { let endpoint = test_endpoint("tacacs-service-authorization-upstream"); let config = service_config(endpoint.clone(), vec![test_server("primary:49")]); - let service = TacacsClientService::new_with_connector(config, connector).unwrap(); + let service = test_service_with_connector(config, connector).unwrap(); let service_task = tokio::spawn(async move { service.serve().await }); tokio::time::sleep(Duration::from_millis(50)).await; @@ -579,6 +680,7 @@ mod tests { vec![test_server("primary:49")], connector, Duration::from_millis(50), + RuntimeHealthPublisher::new(EnabledServices::CLIENT_API), )); let service = GrpcService::new(state, Arc::new(RequestTracker::default())); let request = AuthorizationOperation::builder("admin", 0) @@ -603,4 +705,146 @@ mod tests { assert!(error.retriable); assert!(error.message.contains("simulated failure")); } + + #[tokio::test] + #[cfg_attr(miri, ignore)] // real Unix socket + TCP listener + async fn both_listeners_bind_before_startup_serves_and_cleanup_on_shutdown() { + let endpoint = test_endpoint("tacacs-service-both-listeners"); + let socket_path = match &endpoint { + IpcEndpoint::Unix(path) => path.clone(), + IpcEndpoint::Tcp(_) => unreachable!("Unix test endpoint"), + }; + let mut config = service_config(endpoint, vec![test_server("primary:49")]); + config.enabled_services = EnabledServices::BOTH; + config.proxy_endpoint = + Some(IpcEndpoint::Tcp("127.0.0.1:0".parse().expect("proxy endpoint"))); + let health = RuntimeHealthPublisher::new(EnabledServices::BOTH); + let upstream = Arc::new(FakeConnection { + address: "primary:49".to_owned(), + usable: AtomicBool::new(true), + fail_next_request: AtomicBool::new(false), + }); + let connector = + Arc::new(FakeConnector::new(HashMap::from([(upstream.address.clone(), upstream)]))); + let service = Arc::new( + TacacsClientService::new_with_connector(config, connector, health.clone()) + .expect("service should build"), + ); + let shutdown = ShutdownCoordinator::new(health.clone()); + let task = { + let service = Arc::clone(&service); + let shutdown = shutdown.clone(); + tokio::spawn(async move { service.serve_with_shutdown(&shutdown).await }) + }; + + tokio::time::timeout(Duration::from_secs(2), async { + while health.snapshot().lifecycle() != RuntimeLifecycle::Serving { + tokio::task::yield_now().await; + } + }) + .await + .expect("both listeners should bind"); + assert!(health.snapshot().is_startup_serving()); + assert_eq!(health.snapshot().listener(RuntimeService::ClientApi), ListenerState::Bound,); + assert_eq!(health.snapshot().listener(RuntimeService::TacacsProxy), ListenerState::Bound,); + + shutdown.initiate_shutdown(); + task.await + .expect("task should join") + .expect("shutdown should succeed"); + + assert_eq!(health.snapshot().lifecycle(), RuntimeLifecycle::Stopped); + assert_eq!(health.snapshot().listener(RuntimeService::ClientApi), ListenerState::Stopped,); + assert_eq!(health.snapshot().listener(RuntimeService::TacacsProxy), ListenerState::Stopped,); + assert!(!tokio::fs::try_exists(socket_path) + .await + .expect("inspect socket")); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] // real Unix socket + TCP listener + async fn listener_bind_failure_aborts_sibling_without_publishing_readiness() { + let occupied = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("occupy proxy endpoint"); + let occupied_address = occupied.local_addr().expect("occupied address"); + let endpoint = test_endpoint("tacacs-service-bind-failure"); + let socket_path = match &endpoint { + IpcEndpoint::Unix(path) => path.clone(), + IpcEndpoint::Tcp(_) => unreachable!("Unix test endpoint"), + }; + let mut config = service_config(endpoint, vec![test_server("primary:49")]); + config.enabled_services = EnabledServices::BOTH; + config.proxy_endpoint = Some(IpcEndpoint::Tcp(occupied_address)); + let health = RuntimeHealthPublisher::new(EnabledServices::BOTH); + let upstream = Arc::new(FakeConnection { + address: "primary:49".to_owned(), + usable: AtomicBool::new(true), + fail_next_request: AtomicBool::new(false), + }); + let connector = + Arc::new(FakeConnector::new(HashMap::from([(upstream.address.clone(), upstream)]))); + let service = TacacsClientService::new_with_connector(config, connector, health.clone()) + .expect("service should build"); + let shutdown = ShutdownCoordinator::new(health.clone()); + + let result = service.serve_with_shutdown(&shutdown).await; + + assert!(result.is_err()); + assert_eq!(health.snapshot().lifecycle(), RuntimeLifecycle::Failed); + assert!(!health.snapshot().is_readiness_serving()); + assert_eq!(health.snapshot().listener(RuntimeService::ClientApi), ListenerState::Stopped,); + assert_eq!(health.snapshot().listener(RuntimeService::TacacsProxy), ListenerState::Stopped,); + assert!(!tokio::fs::try_exists(socket_path) + .await + .expect("inspect socket")); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] // real Unix socket + listener drain + async fn shutdown_withdraws_health_before_active_requests_finish_draining() { + let endpoint = test_endpoint("tacacs-service-active-drain"); + let config = service_config(endpoint, vec![test_server("primary:49")]); + let health = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + let upstream = Arc::new(FakeConnection { + address: "primary:49".to_owned(), + usable: AtomicBool::new(true), + fail_next_request: AtomicBool::new(false), + }); + let connector = + Arc::new(FakeConnector::new(HashMap::from([(upstream.address.clone(), upstream)]))); + let service = Arc::new( + TacacsClientService::new_with_connector(config, connector, health.clone()) + .expect("service should build"), + ); + let active_request = service.request_tracker.start_request(); + let shutdown = ShutdownCoordinator::new(health.clone()); + let mut task = { + let service = Arc::clone(&service); + let shutdown = shutdown.clone(); + tokio::spawn(async move { service.serve_with_shutdown(&shutdown).await }) + }; + tokio::time::timeout(Duration::from_secs(2), async { + while health.snapshot().lifecycle() != RuntimeLifecycle::Serving { + tokio::task::yield_now().await; + } + }) + .await + .expect("listener should bind"); + + shutdown.initiate_shutdown(); + tokio::task::yield_now().await; + + assert_eq!(health.snapshot().lifecycle(), RuntimeLifecycle::Draining); + assert!(!health.snapshot().is_liveness_serving()); + assert!(tokio::time::timeout(Duration::from_millis(25), &mut task) + .await + .is_err()); + + drop(active_request); + task.await + .expect("task should join") + .expect("drain should complete"); + assert_eq!(health.snapshot().lifecycle(), RuntimeLifecycle::Stopped); + } } diff --git a/libraries/tacacsrs_agent/src/runtime/health.rs b/libraries/tacacsrs_agent/src/runtime/health.rs new file mode 100644 index 00000000..1bdd2bb0 --- /dev/null +++ b/libraries/tacacsrs_agent/src/runtime/health.rs @@ -0,0 +1,491 @@ +//! Typed runtime health state and publication. +//! +//! The health model describes the agent without embedding a hosting protocol. +//! It contains no free-form messages or configuration values, so every +//! subscriber receives the same secret-free state. +//! +//! ```text +//! datastore ─┐ +//! listeners ─┼─> RuntimeHealthPublisher ─> watch snapshots ─┬─> gRPC health +//! upstreams ─┘ ├─> systemd +//! └─> operator logs +//! ``` +//! +//! Runtime owners publish synchronous state transitions. Consumers subscribe +//! through [`tokio::sync::watch`] and translate immutable snapshots into their +//! platform-specific representation. No lock is exposed or held across an +//! await point. + +use std::collections::BTreeSet; + +use tokio::sync::watch; + +use crate::EnabledServices; + +/// Process lifecycle state. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RuntimeLifecycle { + /// Runtime components are being initialized. + Starting, + /// All startup conditions have completed and listeners are serving. + Serving, + /// Readiness has been withdrawn and listeners are draining active work. + Draining, + /// All runtime work has stopped. + Stopped, + /// An unrecoverable internal failure requires process termination. + Failed, +} + +/// State of the configured datastore snapshot. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum DatastoreState { + /// No complete validated snapshot has been loaded yet. + Loading, + /// The applied snapshot is current with the source. + Current, + /// A previous valid snapshot remains applied after a reload problem. + Stale, + /// The source is unavailable and no current snapshot can be obtained. + Unavailable, +} + +/// A local service listener managed by the runtime. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RuntimeService { + /// The protobuf/gRPC client API listener. + ClientApi, + /// The raw TACACS+ proxy listener. + TacacsProxy, +} + +/// State of a local service listener. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ListenerState { + /// The listener is not enabled for this process. + Disabled, + /// The listener is enabled but binding has not started. + Pending, + /// The listener is creating and configuring its local endpoint. + Binding, + /// The endpoint is bound and accepting work. + Bound, + /// The listener has stopped accepting work. + Stopped, +} + +/// Aggregate result of observed upstream connection attempts. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum UpstreamAvailability { + /// No conclusive connection attempt has been observed for this server set. + Unknown, + /// At least one eligible upstream accepted the latest authoritative attempt. + Available, + /// An authoritative attempt exhausted every eligible upstream. + Unavailable, +} + +/// Typed reason why a live process is operating in a degraded state. +#[derive(Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] +pub enum DegradationReason { + /// The datastore cannot currently be reached. + DatastoreUnavailable, + /// The runtime retained an older known-good datastore snapshot. + DatastoreStale, + /// Continuous datastore notifications are disconnected. + ChangeNotificationsUnavailable, + /// A candidate configuration was rejected while an older snapshot remained active. + CandidateConfigurationRejected, + /// Credential resolution failed for a candidate configuration. + CredentialResolutionFailed, + /// Every eligible upstream failed an authoritative connection attempt. + UpstreamsUnavailable, + /// A runtime invariant or internal component failed. + InternalFailure, +} + +/// Immutable, secret-free view of agent runtime health. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct RuntimeHealthSnapshot { + lifecycle: RuntimeLifecycle, + applied_configuration: bool, + datastore: DatastoreState, + continuous_notifications_connected: bool, + enabled_services: EnabledServices, + client_api_listener: ListenerState, + tacacs_proxy_listener: ListenerState, + eligible_server_count: usize, + upstream_availability: UpstreamAvailability, + degradation_reasons: BTreeSet, +} + +impl RuntimeHealthSnapshot { + /// Returns the process lifecycle state. + #[must_use] + pub const fn lifecycle(&self) -> RuntimeLifecycle { + self.lifecycle + } + + /// Returns whether at least one complete validated configuration was applied. + #[must_use] + pub const fn has_applied_configuration(&self) -> bool { + self.applied_configuration + } + + /// Returns the state of the configured datastore source. + #[must_use] + pub const fn datastore(&self) -> DatastoreState { + self.datastore + } + + /// Returns whether a continuous change-notification subscription is connected. + #[must_use] + pub const fn continuous_notifications_connected(&self) -> bool { + self.continuous_notifications_connected + } + + /// Returns the services enabled for this runtime. + #[must_use] + pub const fn enabled_services(&self) -> EnabledServices { + self.enabled_services + } + + /// Returns the state of a local listener. + #[must_use] + pub const fn listener(&self, service: RuntimeService) -> ListenerState { + match service { + RuntimeService::ClientApi => self.client_api_listener, + RuntimeService::TacacsProxy => self.tacacs_proxy_listener, + } + } + + /// Returns the number of configured servers eligible for runtime operations. + #[must_use] + pub const fn eligible_server_count(&self) -> usize { + self.eligible_server_count + } + + /// Returns aggregate observed upstream availability. + #[must_use] + pub const fn upstream_availability(&self) -> UpstreamAvailability { + self.upstream_availability + } + + /// Returns the typed degradation reasons currently present. + #[must_use] + pub const fn degradation_reasons(&self) -> &BTreeSet { + &self.degradation_reasons + } + + /// Returns whether the initial configuration and listener startup completed. + /// + /// Zero eligible servers does not prevent startup from completing. It does + /// prevent readiness through [`is_readiness_serving`](Self::is_readiness_serving). + #[must_use] + pub fn is_startup_serving(&self) -> bool { + self.accepts_health_requests() + && self.applied_configuration + && self.all_enabled_listeners_bound() + } + + /// Returns whether the process and local health event loop are live. + /// + /// Datastore, credential, and upstream outages do not affect liveness. + #[must_use] + pub fn is_liveness_serving(&self) -> bool { + self.accepts_health_requests() + } + + /// Returns whether business requests may be sent to the agent. + /// + /// Readiness requires completed startup and at least one eligible configured + /// server. Current upstream reachability is deliberately ignored. + #[must_use] + pub fn is_readiness_serving(&self) -> bool { + self.is_startup_serving() && self.eligible_server_count > 0 + } + + fn accepts_health_requests(&self) -> bool { + matches!(self.lifecycle, RuntimeLifecycle::Starting | RuntimeLifecycle::Serving) + } + + fn all_enabled_listeners_bound(&self) -> bool { + (!self.enabled_services.client_api() || self.client_api_listener == ListenerState::Bound) + && (!self.enabled_services.tacacs_proxy() + || self.tacacs_proxy_listener == ListenerState::Bound) + } +} + +/// Single writer for runtime health transitions. +/// +/// Clones share the same watch channel. Mutation methods publish only when the +/// resulting snapshot differs, which keeps health watches free from duplicate +/// transitions. +#[derive(Debug, Clone)] +pub struct RuntimeHealthPublisher { + sender: watch::Sender, +} + +impl RuntimeHealthPublisher { + /// Creates health state for the selected local services. + #[must_use] + pub fn new(enabled_services: EnabledServices) -> Self { + let client_api_listener = initial_listener_state(enabled_services.client_api()); + let tacacs_proxy_listener = initial_listener_state(enabled_services.tacacs_proxy()); + let initial = RuntimeHealthSnapshot { + lifecycle: RuntimeLifecycle::Starting, + applied_configuration: false, + datastore: DatastoreState::Loading, + continuous_notifications_connected: false, + enabled_services, + client_api_listener, + tacacs_proxy_listener, + eligible_server_count: 0, + upstream_availability: UpstreamAvailability::Unknown, + degradation_reasons: BTreeSet::new(), + }; + let (sender, _) = watch::channel(initial); + + Self { sender } + } + + /// Subscribes to the current snapshot and subsequent transitions. + #[must_use] + pub fn subscribe(&self) -> watch::Receiver { + self.sender.subscribe() + } + + /// Returns a clone of the current snapshot without subscribing. + #[must_use] + pub fn snapshot(&self) -> RuntimeHealthSnapshot { + self.sender.borrow().clone() + } + + /// Publishes a process lifecycle transition. + pub fn set_lifecycle(&self, lifecycle: RuntimeLifecycle) { + self.update(|snapshot| snapshot.lifecycle = lifecycle); + } + + /// Publishes whether a complete validated configuration has been applied. + pub fn set_applied_configuration(&self, applied: bool) { + self.update(|snapshot| snapshot.applied_configuration = applied); + } + + /// Publishes the datastore state. + pub fn set_datastore(&self, datastore: DatastoreState) { + self.update(|snapshot| snapshot.datastore = datastore); + } + + /// Publishes continuous change-notification connectivity. + pub fn set_continuous_notifications_connected(&self, connected: bool) { + self.update(|snapshot| snapshot.continuous_notifications_connected = connected); + } + + /// Publishes a local listener transition. + /// + /// Returns `false` without changing the snapshot when the selected listener + /// is disabled. This allows shared lifecycle code to remain race-safe while + /// still surfacing an invalid ownership assumption to its caller. + #[must_use] + pub fn set_listener(&self, service: RuntimeService, state: ListenerState) -> bool { + if !self.service_enabled(service) { + return false; + } + + self.update(|snapshot| match service { + RuntimeService::ClientApi => snapshot.client_api_listener = state, + RuntimeService::TacacsProxy => snapshot.tacacs_proxy_listener = state, + }); + true + } + + /// Publishes the count of configured servers eligible for runtime operations. + pub fn set_eligible_server_count(&self, count: usize) { + self.update(|snapshot| snapshot.eligible_server_count = count); + } + + /// Publishes aggregate observed upstream availability. + pub fn set_upstream_availability(&self, availability: UpstreamAvailability) { + self.update(|snapshot| snapshot.upstream_availability = availability); + } + + /// Adds or removes a typed degradation reason. + pub fn set_degraded(&self, reason: DegradationReason, degraded: bool) { + self.update(|snapshot| { + if degraded { + snapshot.degradation_reasons.insert(reason); + } else { + snapshot.degradation_reasons.remove(&reason); + } + }); + } + + fn service_enabled(&self, service: RuntimeService) -> bool { + let enabled_services = self.sender.borrow().enabled_services; + match service { + RuntimeService::ClientApi => enabled_services.client_api(), + RuntimeService::TacacsProxy => enabled_services.tacacs_proxy(), + } + } + + fn update(&self, mutation: impl FnOnce(&mut RuntimeHealthSnapshot)) { + let mut next = self.snapshot(); + mutation(&mut next); + if next.lifecycle == RuntimeLifecycle::Starting + && next.applied_configuration + && next.all_enabled_listeners_bound() + { + next.lifecycle = RuntimeLifecycle::Serving; + } + if next != *self.sender.borrow() { + self.sender.send_replace(next); + } + } +} + +fn initial_listener_state(enabled: bool) -> ListenerState { + if enabled { + ListenerState::Pending + } else { + ListenerState::Disabled + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn new_publishes_starting_state_for_enabled_services() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::BOTH); + let snapshot = publisher.snapshot(); + + assert_eq!(snapshot.lifecycle(), RuntimeLifecycle::Starting); + assert_eq!(snapshot.datastore(), DatastoreState::Loading); + assert_eq!(snapshot.listener(RuntimeService::ClientApi), ListenerState::Pending); + assert_eq!(snapshot.listener(RuntimeService::TacacsProxy), ListenerState::Pending); + assert_eq!(snapshot.upstream_availability(), UpstreamAvailability::Unknown); + assert!(snapshot.is_liveness_serving()); + assert!(!snapshot.is_startup_serving()); + assert!(!snapshot.is_readiness_serving()); + } + + #[test] + fn startup_waits_for_configuration_and_every_enabled_listener() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::BOTH); + publisher.set_applied_configuration(true); + publisher.set_eligible_server_count(1); + assert!(publisher.set_listener(RuntimeService::ClientApi, ListenerState::Bound)); + + assert!(!publisher.snapshot().is_startup_serving()); + + assert!(publisher.set_listener(RuntimeService::TacacsProxy, ListenerState::Bound)); + let snapshot = publisher.snapshot(); + assert_eq!(snapshot.lifecycle(), RuntimeLifecycle::Serving); + assert!(snapshot.is_startup_serving()); + assert!(snapshot.is_readiness_serving()); + } + + #[test] + fn startup_allows_zero_eligible_servers_but_readiness_does_not() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + publisher.set_applied_configuration(true); + assert!(publisher.set_listener(RuntimeService::ClientApi, ListenerState::Bound)); + + let snapshot = publisher.snapshot(); + assert!(snapshot.is_startup_serving()); + assert!(snapshot.is_liveness_serving()); + assert!(!snapshot.is_readiness_serving()); + assert_eq!(snapshot.listener(RuntimeService::TacacsProxy), ListenerState::Disabled); + } + + #[test] + fn upstream_unavailability_does_not_withdraw_readiness_or_liveness() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + publisher.set_applied_configuration(true); + publisher.set_eligible_server_count(2); + assert!(publisher.set_listener(RuntimeService::ClientApi, ListenerState::Bound)); + publisher.set_upstream_availability(UpstreamAvailability::Unavailable); + publisher.set_degraded(DegradationReason::UpstreamsUnavailable, true); + + let snapshot = publisher.snapshot(); + assert!(snapshot.is_liveness_serving()); + assert!(snapshot.is_readiness_serving()); + assert!(snapshot + .degradation_reasons() + .contains(&DegradationReason::UpstreamsUnavailable)); + } + + #[test] + fn stale_datastore_retains_readiness_from_known_good_configuration() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + publisher.set_applied_configuration(true); + publisher.set_eligible_server_count(1); + assert!(publisher.set_listener(RuntimeService::ClientApi, ListenerState::Bound)); + publisher.set_datastore(DatastoreState::Stale); + publisher.set_degraded(DegradationReason::DatastoreStale, true); + + let snapshot = publisher.snapshot(); + assert!(snapshot.is_readiness_serving()); + assert_eq!(snapshot.datastore(), DatastoreState::Stale); + } + + #[test] + fn draining_withdraws_every_health_view() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + publisher.set_applied_configuration(true); + publisher.set_eligible_server_count(1); + assert!(publisher.set_listener(RuntimeService::ClientApi, ListenerState::Bound)); + publisher.set_lifecycle(RuntimeLifecycle::Serving); + publisher.set_lifecycle(RuntimeLifecycle::Draining); + + let snapshot = publisher.snapshot(); + assert!(!snapshot.is_startup_serving()); + assert!(!snapshot.is_liveness_serving()); + assert!(!snapshot.is_readiness_serving()); + } + + #[tokio::test] + async fn publisher_sends_changed_snapshots_only() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + let mut receiver = publisher.subscribe(); + + publisher.set_datastore(DatastoreState::Loading); + assert!(receiver.has_changed().is_ok_and(|changed| !changed)); + + publisher.set_datastore(DatastoreState::Current); + receiver + .changed() + .await + .expect("publisher should remain alive"); + assert_eq!(receiver.borrow().datastore(), DatastoreState::Current); + } + + #[test] + fn disabled_listener_rejects_transitions() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + + assert!(!publisher.set_listener(RuntimeService::TacacsProxy, ListenerState::Bound)); + assert_eq!( + publisher.snapshot().listener(RuntimeService::TacacsProxy), + ListenerState::Disabled + ); + } + + #[test] + fn snapshot_debug_output_contains_no_configuration_or_secret_values() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::BOTH); + publisher.set_degraded(DegradationReason::CredentialResolutionFailed, true); + let output = format!("{:?}", publisher.snapshot()); + + for forbidden in [ + "redis://", + "unix://", + "192.0.2.10", + "credential-reference", + "test-secret", + ] { + assert!(!output.contains(forbidden), "snapshot exposed {forbidden}"); + } + } +} diff --git a/libraries/tacacsrs_agent/src/runtime/mod.rs b/libraries/tacacsrs_agent/src/runtime/mod.rs index 84720922..6beba081 100644 --- a/libraries/tacacsrs_agent/src/runtime/mod.rs +++ b/libraries/tacacsrs_agent/src/runtime/mod.rs @@ -7,12 +7,17 @@ //! upstream manager. mod client_service; +mod health; mod request_tracker; mod server_catalog; mod shutdown; pub use client_service::TacacsClientService; +pub use health::{ + DatastoreState, DegradationReason, ListenerState, RuntimeHealthPublisher, + RuntimeHealthSnapshot, RuntimeLifecycle, RuntimeService, UpstreamAvailability, +}; pub(crate) use server_catalog::{REQUIRED_SERVER_TYPES, enumerate_supported_servers}; pub(crate) use request_tracker::{RequestGuard, RequestTracker}; -pub(crate) use shutdown::shutdown_signal; +pub(crate) use shutdown::{ListenerRegistration, ShutdownCoordinator, ShutdownReceiver}; diff --git a/libraries/tacacsrs_agent/src/runtime/shutdown.rs b/libraries/tacacsrs_agent/src/runtime/shutdown.rs index 99996043..6600daac 100644 --- a/libraries/tacacsrs_agent/src/runtime/shutdown.rs +++ b/libraries/tacacsrs_agent/src/runtime/shutdown.rs @@ -1,11 +1,125 @@ -//! Process shutdown signal handling for the long-lived agent runtime. +//! Shared process shutdown and listener lifecycle coordination. +//! +//! One coordinator owns the operating-system signal wait. Listener tasks hold +//! cloneable receivers and registration guards: +//! +//! ```text +//! SIGTERM / Ctrl-C +//! | +//! v +//! lifecycle=Draining -> broadcast -> stop accepting -> drain -> Stopped +//! ``` + +use tokio::sync::watch; + +use super::{ListenerState, RuntimeHealthPublisher, RuntimeLifecycle, RuntimeService}; + +/// Broadcasts one ordered shutdown transition to all runtime tasks. +#[derive(Debug, Clone)] +pub(crate) struct ShutdownCoordinator { + sender: watch::Sender, + health: RuntimeHealthPublisher, +} + +impl ShutdownCoordinator { + /// Creates a coordinator for one service runtime. + #[must_use] + pub(crate) fn new(health: RuntimeHealthPublisher) -> Self { + let (sender, _) = watch::channel(false); + Self { sender, health } + } + + /// Returns a receiver for one listener or background task. + #[must_use] + pub(crate) fn subscribe(&self) -> ShutdownReceiver { + ShutdownReceiver { + receiver: self.sender.subscribe(), + } + } + + /// Starts the single operating-system signal monitor. + pub(crate) fn spawn_process_signal_monitor(&self) -> tokio::task::JoinHandle<()> { + let coordinator = self.clone(); + tokio::spawn(async move { + process_shutdown_signal().await; + coordinator.initiate_shutdown(); + }) + } + + /// Withdraws health before broadcasting graceful shutdown. + pub(crate) fn initiate_shutdown(&self) { + self.health.set_lifecycle(RuntimeLifecycle::Draining); + self.sender.send_replace(true); + } + + /// Publishes fatal runtime failure and broadcasts cancellation. + pub(crate) fn fail(&self) { + self.health.set_lifecycle(RuntimeLifecycle::Failed); + self.sender.send_replace(true); + } + + /// Publishes final stopped state after all listener tasks have ended. + pub(crate) fn mark_stopped(&self) { + self.health.set_lifecycle(RuntimeLifecycle::Stopped); + } +} + +/// Cloneable shutdown subscription for one runtime task. +#[derive(Debug, Clone)] +pub(crate) struct ShutdownReceiver { + receiver: watch::Receiver, +} + +impl ShutdownReceiver { + /// Waits until shutdown is broadcast or the coordinator is dropped. + pub(crate) async fn wait(mut self) { + if *self.receiver.borrow() { + return; + } + + while self.receiver.changed().await.is_ok() { + if *self.receiver.borrow() { + return; + } + } + } +} + +/// Publishes listener state and guarantees `Stopped` on every return path. +#[derive(Debug)] +pub(crate) struct ListenerRegistration { + health: RuntimeHealthPublisher, + service: RuntimeService, +} + +impl ListenerRegistration { + /// Registers an enabled listener and publishes `Binding` before bind work. + #[must_use] + pub(crate) fn new(health: RuntimeHealthPublisher, service: RuntimeService) -> Self { + let accepted = health.set_listener(service, ListenerState::Binding); + debug_assert!(accepted, "cannot register a disabled listener"); + Self { health, service } + } + + /// Publishes that the endpoint is configured and accepting work. + pub(crate) fn mark_bound(&self) { + let accepted = self.health.set_listener(self.service, ListenerState::Bound); + debug_assert!(accepted, "cannot bind a disabled listener"); + } +} + +impl Drop for ListenerRegistration { + fn drop(&mut self) { + let accepted = self + .health + .set_listener(self.service, ListenerState::Stopped); + debug_assert!(accepted, "cannot stop a disabled listener"); + } +} /// Waits for a process termination signal that should stop the service from -/// accepting new IPC clients. -/// -/// Unix builds listen for both `SIGTERM` and Ctrl-C. Other platforms fall back -/// to Ctrl-C only. -pub(crate) async fn shutdown_signal() { +/// accepting new local clients. +async fn process_shutdown_signal() { #[cfg(unix)] { use tokio::signal::unix::{SignalKind, signal}; @@ -31,3 +145,36 @@ pub(crate) async fn shutdown_signal() { log::info!("Received Ctrl-C; initiating graceful shutdown"); } } + +#[cfg(test)] +mod tests { + use crate::EnabledServices; + + use super::*; + + #[tokio::test] + async fn shutdown_withdraws_health_before_receiver_completes() { + let health = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + let coordinator = ShutdownCoordinator::new(health.clone()); + let receiver = coordinator.subscribe(); + + coordinator.initiate_shutdown(); + receiver.wait().await; + + assert_eq!(health.snapshot().lifecycle(), RuntimeLifecycle::Draining); + assert!(!health.snapshot().is_liveness_serving()); + } + + #[test] + fn listener_registration_publishes_stopped_on_drop() { + let health = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + let registration = ListenerRegistration::new(health.clone(), RuntimeService::ClientApi); + assert_eq!(health.snapshot().listener(RuntimeService::ClientApi), ListenerState::Binding,); + + registration.mark_bound(); + assert_eq!(health.snapshot().listener(RuntimeService::ClientApi), ListenerState::Bound,); + + drop(registration); + assert_eq!(health.snapshot().listener(RuntimeService::ClientApi), ListenerState::Stopped,); + } +} diff --git a/libraries/tacacsrs_agent/src/services/client_api/health.rs b/libraries/tacacsrs_agent/src/services/client_api/health.rs new file mode 100644 index 00000000..fa556c58 --- /dev/null +++ b/libraries/tacacsrs_agent/src/services/client_api/health.rs @@ -0,0 +1,179 @@ +//! Standard `grpc.health.v1.Health` mapping for the Client API listener. + +use tacacsrs_agent_client::health::{ + LIVENESS_HEALTH_SERVICE, OVERALL_HEALTH_SERVICE, READINESS_HEALTH_SERVICE, + STARTUP_HEALTH_SERVICE, TACACS_AGENT_HEALTH_SERVICE, +}; +use tokio::sync::watch; +use tonic_health::ServingStatus; +use tonic_health::pb::health_server::HealthServer; +use tonic_health::server::{HealthReporter, HealthService}; + +use crate::runtime::{RuntimeHealthSnapshot, ShutdownReceiver}; + +/// Standard health service plus its runtime snapshot reporter. +pub(super) struct StandardHealth { + reporter: HealthReporter, + receiver: watch::Receiver, +} + +impl StandardHealth { + /// Creates and initializes all supported standard health service names. + pub(super) async fn new( + receiver: watch::Receiver, + ) -> (Self, HealthServer) { + let reporter = HealthReporter::new(); + let service = HealthService::from_health_reporter(reporter.clone()); + let health = Self { reporter, receiver }; + health.publish_current().await; + (health, HealthServer::new(service)) + } + + /// Publishes transitions until listener shutdown. + pub(super) async fn run(mut self, shutdown: ShutdownReceiver) { + loop { + tokio::select! { + () = shutdown.clone().wait() => { + self.publish_current().await; + self.close_watches().await; + return; + } + result = self.receiver.changed() => { + if result.is_err() { + return; + } + self.publish_current().await; + } + } + } + } + + async fn publish_current(&self) { + let snapshot = self.receiver.borrow().clone(); + self.set_status(STARTUP_HEALTH_SERVICE, snapshot.is_startup_serving()) + .await; + self.set_status(LIVENESS_HEALTH_SERVICE, snapshot.is_liveness_serving()) + .await; + self.set_status(READINESS_HEALTH_SERVICE, snapshot.is_readiness_serving()) + .await; + self.set_status(OVERALL_HEALTH_SERVICE, snapshot.is_readiness_serving()) + .await; + self.set_status(TACACS_AGENT_HEALTH_SERVICE, snapshot.is_readiness_serving()) + .await; + } + + async fn set_status(&self, service_name: &str, serving: bool) { + let status = if serving { + ServingStatus::Serving + } else { + ServingStatus::NotServing + }; + self.reporter.set_service_status(service_name, status).await; + } + + async fn close_watches(&mut self) { + for service_name in [ + STARTUP_HEALTH_SERVICE, + LIVENESS_HEALTH_SERVICE, + READINESS_HEALTH_SERVICE, + OVERALL_HEALTH_SERVICE, + TACACS_AGENT_HEALTH_SERVICE, + ] { + self.reporter.clear_service_status(service_name).await; + } + } +} + +#[cfg(test)] +mod tests { + use tacacsrs_agent_client::health::{ + LIVENESS_HEALTH_SERVICE, READINESS_HEALTH_SERVICE, STARTUP_HEALTH_SERVICE, + }; + use tonic::Request; + use tonic_health::pb::HealthCheckRequest; + use tonic_health::pb::health_check_response::ServingStatus as WireServingStatus; + use tonic_health::pb::health_server::Health; + + use super::*; + use crate::runtime::{ListenerState, RuntimeHealthPublisher, RuntimeService, ShutdownCoordinator}; + use crate::EnabledServices; + + async fn check(service: &HealthService, service_name: &str) -> WireServingStatus { + let response = service + .check(Request::new(HealthCheckRequest { + service: service_name.to_owned(), + })) + .await + .expect("health check should succeed") + .into_inner(); + WireServingStatus::try_from(response.status).expect("known serving status") + } + + #[tokio::test] + async fn check_maps_startup_liveness_and_readiness_independently() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + let (health, _) = StandardHealth::new(publisher.subscribe()).await; + let service = HealthService::from_health_reporter(health.reporter.clone()); + + assert_eq!(check(&service, STARTUP_HEALTH_SERVICE).await, WireServingStatus::NotServing,); + assert_eq!(check(&service, LIVENESS_HEALTH_SERVICE).await, WireServingStatus::Serving,); + assert_eq!(check(&service, READINESS_HEALTH_SERVICE).await, WireServingStatus::NotServing,); + + publisher.set_applied_configuration(true); + publisher.set_eligible_server_count(1); + assert!(publisher.set_listener(RuntimeService::ClientApi, ListenerState::Bound)); + health.publish_current().await; + + assert_eq!(check(&service, STARTUP_HEALTH_SERVICE).await, WireServingStatus::Serving,); + assert_eq!(check(&service, READINESS_HEALTH_SERVICE).await, WireServingStatus::Serving,); + } + + #[tokio::test] + async fn watch_receives_readiness_transition_and_shutdown_withdrawal() { + let publisher = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + let (health, _) = StandardHealth::new(publisher.subscribe()).await; + let service = HealthService::from_health_reporter(health.reporter.clone()); + let mut stream = service + .watch(Request::new(HealthCheckRequest { + service: READINESS_HEALTH_SERVICE.to_owned(), + })) + .await + .expect("health watch should open") + .into_inner(); + let shutdown = ShutdownCoordinator::new(publisher.clone()); + let bridge = tokio::spawn(health.run(shutdown.subscribe())); + + let initial = tokio_stream::StreamExt::next(&mut stream) + .await + .expect("initial status") + .expect("valid initial status"); + assert_eq!( + WireServingStatus::try_from(initial.status).expect("known status"), + WireServingStatus::NotServing, + ); + + publisher.set_applied_configuration(true); + publisher.set_eligible_server_count(1); + assert!(publisher.set_listener(RuntimeService::ClientApi, ListenerState::Bound)); + let serving = tokio_stream::StreamExt::next(&mut stream) + .await + .expect("serving status") + .expect("valid serving status"); + assert_eq!( + WireServingStatus::try_from(serving.status).expect("known status"), + WireServingStatus::Serving, + ); + + shutdown.initiate_shutdown(); + let withdrawn = tokio_stream::StreamExt::next(&mut stream) + .await + .expect("withdrawn status") + .expect("valid withdrawn status"); + assert_eq!( + WireServingStatus::try_from(withdrawn.status).expect("known status"), + WireServingStatus::NotServing, + ); + bridge.await.expect("bridge should stop"); + assert!(tokio_stream::StreamExt::next(&mut stream).await.is_none()); + } +} diff --git a/libraries/tacacsrs_agent/src/services/client_api/listener/mod.rs b/libraries/tacacsrs_agent/src/services/client_api/listener/mod.rs index 424650cc..d7361fee 100644 --- a/libraries/tacacsrs_agent/src/services/client_api/listener/mod.rs +++ b/libraries/tacacsrs_agent/src/services/client_api/listener/mod.rs @@ -3,8 +3,10 @@ #[cfg(unix)] use anyhow::bail; use tacacsrs_agent_client::IpcEndpoint; +use tokio::sync::watch; use super::ClientApiService; +use crate::runtime::{ListenerRegistration, RuntimeHealthSnapshot, ShutdownReceiver}; use crate::services::ListenerOptions; #[cfg(not(unix))] @@ -18,9 +20,22 @@ pub(crate) async fn serve( endpoint: &IpcEndpoint, service: ClientApiService, options: ListenerOptions, + shutdown: ShutdownReceiver, + registration: ListenerRegistration, + health: watch::Receiver, ) -> anyhow::Result<()> { match endpoint { - IpcEndpoint::Unix(path) => unix::serve(path, service, options.socket_mode()).await, + IpcEndpoint::Unix(path) => { + unix::serve( + path, + service, + options.socket_mode(), + shutdown, + registration, + health, + ) + .await + } IpcEndpoint::Tcp(address) => bail!( "Unix client API endpoints must use a Unix domain socket; TCP endpoint {address} is only supported for the TACACS+ proxy on Unix" ), @@ -44,9 +59,14 @@ pub(crate) async fn serve( endpoint: &IpcEndpoint, service: ClientApiService, _options: ListenerOptions, + shutdown: ShutdownReceiver, + registration: ListenerRegistration, + health: watch::Receiver, ) -> anyhow::Result<()> { match endpoint { - IpcEndpoint::Tcp(address) => tcp::serve(*address, service).await, + IpcEndpoint::Tcp(address) => { + tcp::serve(*address, service, shutdown, registration, health).await + } } } diff --git a/libraries/tacacsrs_agent/src/services/client_api/listener/tcp.rs b/libraries/tacacsrs_agent/src/services/client_api/listener/tcp.rs index 4a73cda1..ba6c9c0d 100644 --- a/libraries/tacacsrs_agent/src/services/client_api/listener/tcp.rs +++ b/libraries/tacacsrs_agent/src/services/client_api/listener/tcp.rs @@ -4,16 +4,24 @@ use std::net::SocketAddr; use anyhow::{Context, bail}; use tacacsrs_agent_client::ipc::tacacs_agent_server::TacacsAgentServer; +use tokio::sync::watch; use tokio_stream::wrappers::TcpListenerStream; -use crate::runtime::shutdown_signal; +use crate::runtime::{ListenerRegistration, RuntimeHealthSnapshot, ShutdownReceiver}; +use crate::services::client_api::health::StandardHealth; use crate::services::client_api::ClientApiService; /// Serves loopback TCP IPC clients until shutdown is requested. /// /// This path exists primarily for non-Unix development workflows where a Unix /// domain socket is not available. -pub(crate) async fn serve(address: SocketAddr, service: ClientApiService) -> anyhow::Result<()> { +pub(crate) async fn serve( + address: SocketAddr, + service: ClientApiService, + shutdown: ShutdownReceiver, + registration: ListenerRegistration, + health: watch::Receiver, +) -> anyhow::Result<()> { if !address.ip().is_loopback() { log::error!("Refusing non-loopback TCP IPC endpoint: {address}"); bail!("TCP IPC endpoint must be loopback-only: {address}"); @@ -23,16 +31,23 @@ pub(crate) async fn serve(address: SocketAddr, service: ClientApiService) -> any .await .with_context(|| format!("Failed to bind TCP IPC endpoint {address}"))?; let incoming = TcpListenerStream::new(listener); + let (standard_health, health_service) = StandardHealth::new(health).await; + let health_task = tokio::spawn(standard_health.run(shutdown.clone())); + registration.mark_bound(); log::info!("Listening for IPC clients on TCP {address}"); tonic::transport::Server::builder() .add_service(TacacsAgentServer::new(service.grpc_service())) - .serve_with_incoming_shutdown(incoming, shutdown_signal()) + .add_service(health_service) + .serve_with_incoming_shutdown(incoming, shutdown.wait()) .await .with_context(|| format!("TCP IPC server {address} failed"))?; log::info!("Shutdown signal received; draining active IPC clients"); service.wait_for_active_requests().await; + health_task + .await + .context("Standard gRPC health bridge failed")?; Ok(()) } diff --git a/libraries/tacacsrs_agent/src/services/client_api/listener/unix.rs b/libraries/tacacsrs_agent/src/services/client_api/listener/unix.rs index 4ba31fcf..2d9cd006 100644 --- a/libraries/tacacsrs_agent/src/services/client_api/listener/unix.rs +++ b/libraries/tacacsrs_agent/src/services/client_api/listener/unix.rs @@ -5,9 +5,11 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, bail}; use tacacsrs_agent_client::ipc::tacacs_agent_server::TacacsAgentServer; +use tokio::sync::watch; use tokio_stream::wrappers::UnixListenerStream; -use crate::runtime::shutdown_signal; +use crate::runtime::{ListenerRegistration, RuntimeHealthSnapshot, ShutdownReceiver}; +use crate::services::client_api::health::StandardHealth; use crate::services::client_api::ClientApiService; /// Serves Unix domain socket IPC clients until shutdown is requested. @@ -18,21 +20,31 @@ pub(crate) async fn serve( path: &Path, service: ClientApiService, socket_mode: u32, + shutdown: ShutdownReceiver, + registration: ListenerRegistration, + health: watch::Receiver, ) -> anyhow::Result<()> { let listener = prepare_unix_listener(path, socket_mode).await?; let socket_guard = UnixSocketCleanupGuard::new(path); let incoming = UnixListenerStream::new(listener); + let (standard_health, health_service) = StandardHealth::new(health).await; + let health_task = tokio::spawn(standard_health.run(shutdown.clone())); + registration.mark_bound(); log::info!("Listening for IPC clients on Unix socket {}", path.display()); tonic::transport::Server::builder() .add_service(TacacsAgentServer::new(service.grpc_service())) - .serve_with_incoming_shutdown(incoming, shutdown_signal()) + .add_service(health_service) + .serve_with_incoming_shutdown(incoming, shutdown.wait()) .await .with_context(|| format!("Unix IPC server {} failed", path.display()))?; log::info!("Shutdown signal received; draining active IPC clients"); service.wait_for_active_requests().await; + health_task + .await + .context("Standard gRPC health bridge failed")?; socket_guard.cleanup("Unix socket").await?; Ok(()) } diff --git a/libraries/tacacsrs_agent/src/services/client_api/mod.rs b/libraries/tacacsrs_agent/src/services/client_api/mod.rs index 00ca61ac..ed9539aa 100644 --- a/libraries/tacacsrs_agent/src/services/client_api/mod.rs +++ b/libraries/tacacsrs_agent/src/services/client_api/mod.rs @@ -7,6 +7,7 @@ //! manager. mod grpc; +mod health; pub(crate) mod listener; mod service; mod upstream_bridge; diff --git a/libraries/tacacsrs_agent/src/services/client_api/service.rs b/libraries/tacacsrs_agent/src/services/client_api/service.rs index a10d3131..880c3f2e 100644 --- a/libraries/tacacsrs_agent/src/services/client_api/service.rs +++ b/libraries/tacacsrs_agent/src/services/client_api/service.rs @@ -6,7 +6,9 @@ use tacacsrs_agent_client::IpcEndpoint; use super::GrpcService; use super::listener; -use crate::runtime::RequestTracker; +use crate::runtime::{ + ListenerRegistration, RequestTracker, RuntimeHealthPublisher, RuntimeService, ShutdownReceiver, +}; use crate::services::ListenerOptions; use crate::upstream::manager::UpstreamManager; @@ -34,8 +36,13 @@ impl ClientApiService { &self, endpoint: &IpcEndpoint, options: ListenerOptions, + shutdown: ShutdownReceiver, + health: RuntimeHealthPublisher, ) -> anyhow::Result<()> { - listener::serve(endpoint, self.clone(), options).await + let health_receiver = health.subscribe(); + let registration = ListenerRegistration::new(health, RuntimeService::ClientApi); + listener::serve(endpoint, self.clone(), options, shutdown, registration, health_receiver) + .await } #[cfg(unix)] diff --git a/libraries/tacacsrs_agent/src/services/client_api/upstream_bridge/mod.rs b/libraries/tacacsrs_agent/src/services/client_api/upstream_bridge/mod.rs index a86b982e..04f38aea 100644 --- a/libraries/tacacsrs_agent/src/services/client_api/upstream_bridge/mod.rs +++ b/libraries/tacacsrs_agent/src/services/client_api/upstream_bridge/mod.rs @@ -38,10 +38,11 @@ mod tests { use tacacsrs_config::TacacsPlusServer; use super::UpstreamBridge; - use crate::runtime::REQUIRED_SERVER_TYPES; + use crate::runtime::{REQUIRED_SERVER_TYPES, RuntimeHealthPublisher}; use crate::test_support::{FakeConnection, FakeConnector, build_authorization_request}; use crate::upstream::UpstreamConnector; use crate::upstream::manager::UpstreamManager; + use crate::EnabledServices; fn test_server(address: &str) -> TacacsPlusServer { let (host, port) = match address.rsplit_once(':') { @@ -84,6 +85,7 @@ mod tests { vec![test_server("server:49")], Arc::clone(&connector) as Arc, Duration::from_millis(200), + RuntimeHealthPublisher::new(EnabledServices::CLIENT_API), )); let bridge = UpstreamBridge::new(Arc::clone(&state)); @@ -118,6 +120,7 @@ mod tests { vec![test_server("primary:49"), test_server("secondary:49")], Arc::clone(&connector) as Arc, Duration::from_millis(200), + RuntimeHealthPublisher::new(EnabledServices::CLIENT_API), )); let bridge = UpstreamBridge::new(Arc::clone(&state)); @@ -141,6 +144,7 @@ mod tests { Vec::new(), Arc::new(FakeConnector::new(HashMap::new())) as Arc, Duration::from_millis(200), + RuntimeHealthPublisher::new(EnabledServices::CLIENT_API), )); let bridge = UpstreamBridge::new(Arc::clone(&state)); diff --git a/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/mod.rs b/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/mod.rs index 5619db88..cf9eabf1 100644 --- a/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/mod.rs +++ b/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/mod.rs @@ -4,7 +4,7 @@ use tacacsrs_agent_client::IpcEndpoint; use tokio::io::{AsyncRead, AsyncWrite}; use super::TacacsProxyService; -use crate::runtime::shutdown_signal; +use crate::runtime::{ListenerRegistration, ShutdownReceiver}; use crate::services::ListenerOptions; mod tcp; @@ -16,10 +16,14 @@ pub(super) async fn serve( endpoint: &IpcEndpoint, service: TacacsProxyService, options: ListenerOptions, + shutdown: ShutdownReceiver, + registration: ListenerRegistration, ) -> anyhow::Result<()> { match endpoint { - IpcEndpoint::Unix(path) => unix::serve(path, service, options.socket_mode()).await, - IpcEndpoint::Tcp(address) => tcp::serve(*address, service).await, + IpcEndpoint::Unix(path) => { + unix::serve(path, service, options.socket_mode(), shutdown, registration).await + } + IpcEndpoint::Tcp(address) => tcp::serve(*address, service, shutdown, registration).await, } } @@ -28,9 +32,11 @@ pub(super) async fn serve( endpoint: &IpcEndpoint, service: TacacsProxyService, _options: ListenerOptions, + shutdown: ShutdownReceiver, + registration: ListenerRegistration, ) -> anyhow::Result<()> { match endpoint { - IpcEndpoint::Tcp(address) => tcp::serve(*address, service).await, + IpcEndpoint::Tcp(address) => tcp::serve(*address, service, shutdown, registration).await, } } @@ -38,12 +44,13 @@ async fn accept_loop( listener: Listener, service: TacacsProxyService, endpoint_label: String, + shutdown: ShutdownReceiver, ) -> anyhow::Result<()> where Listener: ProxyListener, Stream: AsyncRead + AsyncWrite + Unpin + Send + 'static, { - let mut shutdown = Box::pin(shutdown_signal()); + let mut shutdown = Box::pin(shutdown.wait()); loop { tokio::select! { diff --git a/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/tcp.rs b/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/tcp.rs index 1259210d..1abcda39 100644 --- a/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/tcp.rs +++ b/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/tcp.rs @@ -5,9 +5,15 @@ use std::net::SocketAddr; use anyhow::{Context, bail}; use super::{ProxyListener, accept_loop}; +use crate::runtime::{ListenerRegistration, ShutdownReceiver}; use crate::services::tacacs_proxy::TacacsProxyService; -pub(super) async fn serve(address: SocketAddr, service: TacacsProxyService) -> anyhow::Result<()> { +pub(super) async fn serve( + address: SocketAddr, + service: TacacsProxyService, + shutdown: ShutdownReceiver, + registration: ListenerRegistration, +) -> anyhow::Result<()> { if !address.ip().is_loopback() { log::error!("Refusing non-loopback TCP TACACS+ proxy endpoint: {address}"); bail!("TCP TACACS+ proxy endpoint must be loopback-only: {address}"); @@ -19,9 +25,10 @@ pub(super) async fn serve(address: SocketAddr, service: TacacsProxyService) -> a let local_address = listener .local_addr() .with_context(|| format!("Failed to inspect TCP TACACS+ proxy endpoint {address}"))?; + registration.mark_bound(); log::info!("Listening for TACACS+ proxy clients on TCP {local_address}"); - accept_loop(listener, service, format!("TCP {local_address}")).await + accept_loop(listener, service, format!("TCP {local_address}"), shutdown).await } #[async_trait::async_trait] diff --git a/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/unix.rs b/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/unix.rs index 681f4b21..a1588834 100644 --- a/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/unix.rs +++ b/libraries/tacacsrs_agent/src/services/tacacs_proxy/listener/unix.rs @@ -5,6 +5,7 @@ use std::path::Path; use anyhow::Context; use super::{ProxyListener, accept_loop}; +use crate::runtime::{ListenerRegistration, ShutdownReceiver}; use crate::services::client_api::listener as client_api_listener; use crate::services::tacacs_proxy::TacacsProxyService; @@ -12,12 +13,16 @@ pub(super) async fn serve( path: &Path, service: TacacsProxyService, socket_mode: u32, + shutdown: ShutdownReceiver, + registration: ListenerRegistration, ) -> anyhow::Result<()> { let listener = client_api_listener::prepare_unix_listener(path, socket_mode).await?; let socket_guard = client_api_listener::UnixSocketCleanupGuard::new(path); + registration.mark_bound(); log::info!("Listening for TACACS+ proxy clients on Unix socket {}", path.display()); - let result = accept_loop(listener, service, format!("Unix socket {}", path.display())).await; + let result = + accept_loop(listener, service, format!("Unix socket {}", path.display()), shutdown).await; socket_guard.cleanup("TACACS+ proxy Unix socket").await?; result } diff --git a/libraries/tacacsrs_agent/src/services/tacacs_proxy/service.rs b/libraries/tacacsrs_agent/src/services/tacacs_proxy/service.rs index 00fd8e73..2c6399dc 100644 --- a/libraries/tacacsrs_agent/src/services/tacacs_proxy/service.rs +++ b/libraries/tacacsrs_agent/src/services/tacacs_proxy/service.rs @@ -9,7 +9,10 @@ use tokio::sync::RwLock; use super::listener; use super::upstream_bridge::UpstreamBridge; use crate::config::ProxyDownstreamObfuscation; -use crate::runtime::{RequestGuard, RequestTracker}; +use crate::runtime::{ + ListenerRegistration, RequestGuard, RequestTracker, RuntimeHealthPublisher, RuntimeService, + ShutdownReceiver, +}; use crate::services::ListenerOptions; use crate::upstream::manager::UpstreamManager; @@ -38,8 +41,11 @@ impl TacacsProxyService { &self, endpoint: &IpcEndpoint, options: ListenerOptions, + shutdown: ShutdownReceiver, + health: RuntimeHealthPublisher, ) -> anyhow::Result<()> { - listener::serve(endpoint, self.clone(), options).await + let registration = ListenerRegistration::new(health, RuntimeService::TacacsProxy); + listener::serve(endpoint, self.clone(), options, shutdown, registration).await } pub(super) async fn handle_connection( diff --git a/libraries/tacacsrs_agent/src/upstream/manager/availability.rs b/libraries/tacacsrs_agent/src/upstream/manager/availability.rs new file mode 100644 index 00000000..86678a74 --- /dev/null +++ b/libraries/tacacsrs_agent/src/upstream/manager/availability.rs @@ -0,0 +1,126 @@ +//! Race-safe aggregate upstream availability observations. + +use std::sync::Mutex; + +use crate::runtime::{DegradationReason, RuntimeHealthPublisher, UpstreamAvailability}; + +/// Identity of one aggregate connection attempt within a server-set generation. +#[derive(Debug, Clone, Copy)] +pub(super) struct AvailabilityAttempt { + generation: u64, + sequence: u64, +} + +#[derive(Debug, Default)] +struct ObservationState { + generation: u64, + next_sequence: u64, + last_observed_sequence: u64, +} + +/// Publishes only observations that still belong to the current server set and +/// are newer than the last completed authoritative attempt. +pub(super) struct AvailabilityTracker { + health: RuntimeHealthPublisher, + state: Mutex, +} + +impl AvailabilityTracker { + pub(super) fn new(health: RuntimeHealthPublisher) -> Self { + Self { + health, + state: Mutex::new(ObservationState::default()), + } + } + + pub(super) fn begin_attempt(&self) -> AvailabilityAttempt { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.next_sequence = state.next_sequence.saturating_add(1); + AvailabilityAttempt { + generation: state.generation, + sequence: state.next_sequence, + } + } + + pub(super) fn available(&self, attempt: AvailabilityAttempt) { + self.observe(attempt, UpstreamAvailability::Available); + } + + pub(super) fn unavailable(&self, attempt: AvailabilityAttempt) { + self.observe(attempt, UpstreamAvailability::Unavailable); + } + + pub(super) fn reset(&self) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.generation = state.generation.saturating_add(1); + state.next_sequence = 0; + state.last_observed_sequence = 0; + self.health + .set_upstream_availability(UpstreamAvailability::Unknown); + self.health + .set_degraded(DegradationReason::UpstreamsUnavailable, false); + } + + fn observe(&self, attempt: AvailabilityAttempt, availability: UpstreamAvailability) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if attempt.generation != state.generation { + return; + } + if attempt.sequence <= state.last_observed_sequence { + return; + } + + state.last_observed_sequence = attempt.sequence; + self.health.set_upstream_availability(availability); + self.health.set_degraded( + DegradationReason::UpstreamsUnavailable, + availability == UpstreamAvailability::Unavailable, + ); + } +} + +#[cfg(test)] +mod tests { + use crate::EnabledServices; + + use super::*; + + #[test] + fn newer_success_cannot_be_overwritten_by_older_failure() { + let health = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + let tracker = AvailabilityTracker::new(health.clone()); + let older = tracker.begin_attempt(); + let newer = tracker.begin_attempt(); + + tracker.available(newer); + tracker.unavailable(older); + + assert_eq!(health.snapshot().upstream_availability(), UpstreamAvailability::Available,); + } + + #[test] + fn server_set_reset_invalidates_in_flight_attempts() { + let health = RuntimeHealthPublisher::new(EnabledServices::CLIENT_API); + let tracker = AvailabilityTracker::new(health.clone()); + let old_generation = tracker.begin_attempt(); + tracker.unavailable(old_generation); + + tracker.reset(); + tracker.available(old_generation); + + assert_eq!(health.snapshot().upstream_availability(), UpstreamAvailability::Unknown,); + assert!(!health + .snapshot() + .degradation_reasons() + .contains(&DegradationReason::UpstreamsUnavailable)); + } +} diff --git a/libraries/tacacsrs_agent/src/upstream/manager/mod.rs b/libraries/tacacsrs_agent/src/upstream/manager/mod.rs index 570c424c..8cb7a1ab 100644 --- a/libraries/tacacsrs_agent/src/upstream/manager/mod.rs +++ b/libraries/tacacsrs_agent/src/upstream/manager/mod.rs @@ -24,15 +24,17 @@ use std::sync::atomic::Ordering; use anyhow::bail; use tacacsrs_config::{TacacsPlusServer, TacacsPlusServerExt}; +use self::availability::AvailabilityTracker; use self::server_set::{ServerSet, servers_equivalent}; use self::server_slot::ServerSlot; -use crate::runtime::REQUIRED_SERVER_TYPES; +use crate::runtime::{REQUIRED_SERVER_TYPES, RuntimeHealthPublisher}; use crate::upstream::{UpstreamConnection, UpstreamConnector}; pub(crate) use self::server_set::BoundServer; mod server_set; mod server_slot; +mod availability; /// Shared runtime state for all IPC client handlers spawned by the listener. /// @@ -57,6 +59,8 @@ pub(crate) struct UpstreamManager { connector: Arc, /// Interval between preferred-server recovery probes. preferred_probe_interval: std::time::Duration, + /// Race-safe aggregate upstream availability publisher. + availability: AvailabilityTracker, } impl UpstreamManager { @@ -69,6 +73,7 @@ impl UpstreamManager { servers: Vec, connector: Arc, preferred_probe_interval: std::time::Duration, + health: RuntimeHealthPublisher, ) -> Self { debug_assert!( servers @@ -85,6 +90,7 @@ impl UpstreamManager { server_set: StdRwLock::new(Arc::new(ServerSet::new(servers, 0))), connector, preferred_probe_interval, + availability: AvailabilityTracker::new(health), } } @@ -116,6 +122,7 @@ impl UpstreamManager { ); return; } + let availability_attempt = self.availability.begin_attempt(); let start_index = *server_set.active_index.read().await; for offset in 0..server_set.server_count() { @@ -123,6 +130,7 @@ impl UpstreamManager { match self.ensure_connection(&server_set.servers[index]).await { Ok(connection) => { *server_set.active_index.write().await = index; + self.availability.available(availability_attempt); log::info!( "Initialized startup upstream connection using {}", connection.server_address() @@ -141,6 +149,7 @@ impl UpstreamManager { log::warn!( "Startup did not find a responsive TACACS+ server; requests will retry on demand" ); + self.availability.unavailable(availability_attempt); } /// Returns the number of configured upstream TACACS+ servers. @@ -161,6 +170,12 @@ impl UpstreamManager { servers: Vec, ) -> anyhow::Result<()> { let previous = self.current_server_set(); + let materially_changed = previous.server_count() != servers.len() + || previous + .servers + .iter() + .zip(&servers) + .any(|(old, new)| !servers_equivalent(&old.server, new)); let previous_active_name = if previous.server_count() == 0 { None } else { @@ -197,6 +212,9 @@ impl UpstreamManager { .unwrap_or_else(std::sync::PoisonError::into_inner); *current = Arc::clone(&new_set); } + if materially_changed { + self.availability.reset(); + } if new_set.server_count() == 0 { log::warn!( @@ -254,8 +272,10 @@ impl UpstreamManager { server_set.servers[active_index].server.socket_address(), ); + let availability_attempt = state.availability.begin_attempt(); match state.ensure_connection(&server_set.servers[0]).await { Ok(connection) => { + state.availability.available(availability_attempt); log::info!( "Preferred TACACS+ server {} recovered; routing new sessions back to it", connection.server_address() @@ -303,6 +323,7 @@ impl UpstreamManager { "No TACACS+ servers are configured yet that support authentication, authorization, and accounting; waiting for initial configuration" ); } + let availability_attempt = self.availability.begin_attempt(); let start_index = *server_set.active_index.read().await; for offset in 0..server_set.server_count() { @@ -310,6 +331,7 @@ impl UpstreamManager { match self.ensure_connection(&server_set.servers[index]).await { Ok(connection) => { *server_set.active_index.write().await = index; + self.availability.available(availability_attempt); return Ok(BoundServer { server_set, index, @@ -329,6 +351,7 @@ impl UpstreamManager { log::error!( "All configured TACACS+ servers are currently non-responsive; failing IPC request" ); + self.availability.unavailable(availability_attempt); bail!("No responsive TACACS+ servers are currently available"); } @@ -470,7 +493,8 @@ mod tests { use tacacsrs_config::TacacsPlusServer; use super::UpstreamManager; - use crate::runtime::REQUIRED_SERVER_TYPES; + use crate::runtime::{REQUIRED_SERVER_TYPES, RuntimeHealthPublisher}; + use crate::EnabledServices; use crate::test_support::{FakeConnection, FakeConnector}; use crate::upstream::UpstreamConnector; @@ -497,6 +521,10 @@ mod tests { } } + fn test_health() -> RuntimeHealthPublisher { + RuntimeHealthPublisher::new(EnabledServices::CLIENT_API) + } + #[tokio::test] #[cfg_attr(miri, ignore)] // tokio spawn/time not supported async fn test_server_selection_wraps_to_later_server() { @@ -530,6 +558,7 @@ mod tests { ], connector, Duration::from_millis(25), + test_health(), ); let bound = state.bind_server_for_new_session().await.unwrap(); @@ -569,6 +598,7 @@ mod tests { ], Arc::clone(&connector) as Arc, Duration::from_millis(25), + test_health(), ); state.warm_connections().await; @@ -605,6 +635,7 @@ mod tests { vec![test_server("server-a:49")], Arc::clone(&connector) as Arc, Duration::from_millis(25), + test_health(), ); state.warm_connections().await; @@ -644,6 +675,7 @@ mod tests { vec![test_server("server-a:49")], Arc::clone(&connector) as Arc, Duration::from_millis(25), + test_health(), ); state.warm_connections().await; @@ -662,6 +694,142 @@ mod tests { assert!(!first.usable.load(Ordering::Relaxed)); } + #[tokio::test(start_paused = true)] + #[cfg_attr(miri, ignore)] // tokio spawn/time not supported + async fn preferred_probe_started_with_zero_servers_recovers_after_reload() { + let preferred = Arc::new(FakeConnection { + address: "server-a:49".to_owned(), + usable: AtomicBool::new(false), + fail_next_request: AtomicBool::new(false), + }); + let backup = Arc::new(FakeConnection { + address: "server-b:49".to_owned(), + usable: AtomicBool::new(true), + fail_next_request: AtomicBool::new(false), + }); + let connector = Arc::new(FakeConnector::new(HashMap::from([ + (preferred.address.clone(), Arc::clone(&preferred)), + (backup.address.clone(), Arc::clone(&backup)), + ]))); + let health = test_health(); + let state = Arc::new(UpstreamManager::new( + Vec::new(), + Arc::clone(&connector) as Arc, + Duration::from_millis(25), + health.clone(), + )); + let probe = state.spawn_preferred_probe(); + tokio::task::yield_now().await; + + state + .reload_servers(vec![test_server("server-a:49"), test_server("server-b:49")]) + .await + .expect("reload should succeed"); + let failed_over = state + .bind_server_for_new_session() + .await + .expect("backup should bind"); + assert_eq!(failed_over.connection.server_address(), backup.address); + assert_eq!( + health.snapshot().upstream_availability(), + crate::UpstreamAvailability::Available, + ); + + tokio::time::advance(Duration::from_millis(25)).await; + tokio::task::yield_now().await; + assert_eq!( + health.snapshot().upstream_availability(), + crate::UpstreamAvailability::Available, + ); + preferred.usable.store(true, Ordering::Relaxed); + + tokio::time::advance(Duration::from_millis(25)).await; + tokio::task::yield_now().await; + let recovered = state + .bind_server_for_new_session() + .await + .expect("preferred should recover"); + assert_eq!(recovered.connection.server_address(), preferred.address); + assert_eq!(connector.connect_attempts_for(&preferred.address).await, 3); + + tokio::time::advance(Duration::from_millis(100)).await; + tokio::task::yield_now().await; + assert_eq!(connector.connect_attempts_for(&preferred.address).await, 3); + probe.abort(); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] // tokio sync not supported + async fn aggregate_exhaustion_marks_upstreams_unavailable() { + let first = Arc::new(FakeConnection { + address: "server-a:49".to_owned(), + usable: AtomicBool::new(false), + fail_next_request: AtomicBool::new(false), + }); + let second = Arc::new(FakeConnection { + address: "server-b:49".to_owned(), + usable: AtomicBool::new(false), + fail_next_request: AtomicBool::new(false), + }); + let connector = Arc::new(FakeConnector::new(HashMap::from([ + (first.address.clone(), first), + (second.address.clone(), second), + ]))); + let health = test_health(); + let state = UpstreamManager::new( + vec![test_server("server-a:49"), test_server("server-b:49")], + connector, + Duration::from_millis(25), + health.clone(), + ); + + let result = state.bind_server_for_new_session().await; + + assert!(result.is_err()); + assert_eq!( + health.snapshot().upstream_availability(), + crate::UpstreamAvailability::Unavailable, + ); + assert!(health + .snapshot() + .degradation_reasons() + .contains(&crate::DegradationReason::UpstreamsUnavailable)); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] // tokio sync not supported + async fn material_server_reload_resets_availability_to_unknown() { + let connection = Arc::new(FakeConnection { + address: "server-a:49".to_owned(), + usable: AtomicBool::new(true), + fail_next_request: AtomicBool::new(false), + }); + let connector = + Arc::new(FakeConnector::new(HashMap::from([(connection.address.clone(), connection)]))); + let health = test_health(); + let state = UpstreamManager::new( + vec![test_server("server-a:49")], + connector, + Duration::from_millis(25), + health.clone(), + ); + state + .bind_server_for_new_session() + .await + .expect("server should bind"); + assert_eq!( + health.snapshot().upstream_availability(), + crate::UpstreamAvailability::Available, + ); + + state + .reload_servers(vec![test_server("server-a:49"), test_server("server-b:49")]) + .await + .expect("reload should succeed"); + + assert_eq!(health.snapshot().upstream_availability(), crate::UpstreamAvailability::Unknown,); + } + #[tokio::test] #[cfg_attr(miri, ignore)] // tokio spawn/time not supported async fn test_concurrent_failover_coalesces_connection_attempts() { @@ -688,6 +856,7 @@ mod tests { vec![test_server("server-a:49"), test_server("server-b:49")], Arc::clone(&connector) as Arc, Duration::from_millis(200), + test_health(), )); let mut tasks = Vec::new(); diff --git a/libraries/tacacsrs_agent_client/Cargo.toml b/libraries/tacacsrs_agent_client/Cargo.toml index 1e68cfba..8a90f49e 100644 --- a/libraries/tacacsrs_agent_client/Cargo.toml +++ b/libraries/tacacsrs_agent_client/Cargo.toml @@ -15,6 +15,7 @@ hyper-util = "0.1.20" prost = "0.14.3" tokio = { version = "1.52.3", features = ["full"] } tonic = "0.14.6" +tonic-health = "0.14.6" tonic-prost = "0.14.6" tower = { version = "0.5.3", features = ["util"] } diff --git a/libraries/tacacsrs_agent_client/src/client.rs b/libraries/tacacsrs_agent_client/src/client.rs index 70d54107..a5c7ee7c 100644 --- a/libraries/tacacsrs_agent_client/src/client.rs +++ b/libraries/tacacsrs_agent_client/src/client.rs @@ -18,13 +18,7 @@ //! connection overhead. use anyhow::{Context, anyhow}; -#[cfg(unix)] -use http::Uri; -#[cfg(unix)] -use hyper_util::rt::TokioIo; -use tonic::transport::{Channel, Endpoint}; -#[cfg(unix)] -use tower::service_fn; +use tonic::transport::Channel; use crate::ipc; use crate::ipc::tacacs_agent_client::TacacsAgentClient; @@ -32,11 +26,9 @@ use crate::protocol::{ AccountingOperation, AccountingOperationResponse, AuthorizationOperation, AuthorizationOperationResponse, ServiceError, }; +use crate::endpoint::connect_channel; use crate::IpcEndpoint; -#[cfg(unix)] -const UDS_GRPC_CONNECT_URI: &str = "http://[::]:50051"; - /// Convenience wrapper for making local IPC calls to the central service. /// /// `ServiceClient` holds a persistent gRPC [`Channel`] that is established once @@ -107,32 +99,7 @@ impl ServiceClient { /// Returns an error if the IPC connection cannot be established (socket /// missing, service not running, etc.). pub async fn connect(endpoint: IpcEndpoint) -> anyhow::Result { - let channel = match &endpoint { - #[cfg(unix)] - IpcEndpoint::Unix(path) => { - let path = path.clone(); - let connect_path = path.clone(); - Endpoint::try_from(UDS_GRPC_CONNECT_URI) - .context("Failed to build Unix IPC gRPC endpoint")? - .connect_with_connector(service_fn(move |_: Uri| { - let path = connect_path.clone(); - async move { - tokio::net::UnixStream::connect(path) - .await - .map(TokioIo::new) - } - })) - .await - .with_context(|| { - format!("Failed to connect to service socket {}", path.display()) - })? - } - IpcEndpoint::Tcp(address) => Endpoint::from_shared(format!("http://{address}")) - .context("Failed to build TCP IPC gRPC endpoint")? - .connect() - .await - .with_context(|| format!("Failed to connect to service endpoint {address}"))?, - }; + let channel = connect_channel(&endpoint).await?; Ok(Self { channel }) } diff --git a/libraries/tacacsrs_agent_client/src/endpoint.rs b/libraries/tacacsrs_agent_client/src/endpoint.rs index d0918a25..95f12800 100644 --- a/libraries/tacacsrs_agent_client/src/endpoint.rs +++ b/libraries/tacacsrs_agent_client/src/endpoint.rs @@ -19,6 +19,16 @@ use std::path::PathBuf; use std::str::FromStr; use anyhow::{Context, bail}; +#[cfg(unix)] +use http::Uri; +#[cfg(unix)] +use hyper_util::rt::TokioIo; +use tonic::transport::{Channel, Endpoint}; +#[cfg(unix)] +use tower::service_fn; + +#[cfg(unix)] +const UDS_GRPC_CONNECT_URI: &str = "http://[::]:50051"; /// Local IPC endpoint used between local consumers and the central service. /// @@ -110,6 +120,39 @@ impl FromStr for IpcEndpoint { } } +/// Establishes a reusable gRPC channel to a local IPC endpoint. +/// +/// # Errors +/// +/// Returns an error when the endpoint cannot be represented as a gRPC +/// transport or the local socket cannot be reached. +pub async fn connect_channel(endpoint: &IpcEndpoint) -> anyhow::Result { + match endpoint { + #[cfg(unix)] + IpcEndpoint::Unix(path) => { + let path = path.clone(); + let connect_path = path.clone(); + Endpoint::try_from(UDS_GRPC_CONNECT_URI) + .context("Failed to build Unix IPC gRPC endpoint")? + .connect_with_connector(service_fn(move |_: Uri| { + let path = connect_path.clone(); + async move { + tokio::net::UnixStream::connect(path) + .await + .map(TokioIo::new) + } + })) + .await + .with_context(|| format!("Failed to connect to service socket {}", path.display())) + } + IpcEndpoint::Tcp(address) => Endpoint::from_shared(format!("http://{address}")) + .context("Failed to build TCP IPC gRPC endpoint")? + .connect() + .await + .with_context(|| format!("Failed to connect to service endpoint {address}")), + } +} + #[cfg(test)] mod tests { use super::IpcEndpoint; diff --git a/libraries/tacacsrs_agent_client/src/health.rs b/libraries/tacacsrs_agent_client/src/health.rs new file mode 100644 index 00000000..1d644ce7 --- /dev/null +++ b/libraries/tacacsrs_agent_client/src/health.rs @@ -0,0 +1,61 @@ +//! Stable service names used with the standard gRPC health protocol. + +use tonic_health::pb::HealthCheckRequest; +use tonic_health::pb::health_check_response::ServingStatus; +use tonic_health::pb::health_client::HealthClient as GrpcHealthClient; + +use crate::endpoint::connect_channel; +use crate::IpcEndpoint; + +/// Overall health service name. The empty name follows the gRPC health +/// protocol convention and maps to agent readiness. +pub const OVERALL_HEALTH_SERVICE: &str = ""; + +/// Business RPC service represented by agent readiness. +pub const TACACS_AGENT_HEALTH_SERVICE: &str = "tacacsrs.agent.v1.TacacsAgent"; + +/// Startup probe service name. +pub const STARTUP_HEALTH_SERVICE: &str = "tacacsrs.agent.health.v1.Startup"; + +/// Liveness probe service name. +pub const LIVENESS_HEALTH_SERVICE: &str = "tacacsrs.agent.health.v1.Liveness"; + +/// Readiness probe service name. +pub const READINESS_HEALTH_SERVICE: &str = "tacacsrs.agent.health.v1.Readiness"; + +/// Standard gRPC health client over the same local transport as [`crate::ServiceClient`]. +#[derive(Debug, Clone)] +pub struct HealthClient { + client: GrpcHealthClient, +} + +impl HealthClient { + /// Connects to the standard health service on a local IPC endpoint. + /// + /// # Errors + /// + /// Returns an error when the local endpoint cannot be reached. + pub async fn connect(endpoint: &IpcEndpoint) -> anyhow::Result { + let channel = connect_channel(endpoint).await?; + Ok(Self { + client: GrpcHealthClient::new(channel), + }) + } + + /// Checks one standard gRPC health service name. + /// + /// # Errors + /// + /// Returns the standard gRPC status for unknown names and protocol or + /// transport failures. + pub async fn check(&mut self, service_name: &str) -> Result { + let response = self + .client + .check(HealthCheckRequest { + service: service_name.to_owned(), + }) + .await? + .into_inner(); + Ok(ServingStatus::try_from(response.status).unwrap_or(ServingStatus::Unknown)) + } +} diff --git a/libraries/tacacsrs_agent_client/src/lib.rs b/libraries/tacacsrs_agent_client/src/lib.rs index 414367ce..429cd103 100644 --- a/libraries/tacacsrs_agent_client/src/lib.rs +++ b/libraries/tacacsrs_agent_client/src/lib.rs @@ -10,6 +10,9 @@ pub mod client; mod endpoint; +/// Stable names used by standard gRPC health checks. +pub mod health; + /// Generated protobuf/gRPC types for the local IPC transport. /// /// The `.proto` file @@ -32,6 +35,7 @@ pub mod protocol; pub use client::ServiceClient; pub use endpoint::IpcEndpoint; +pub use health::HealthClient; pub use protocol::{ AccountingOperation, AccountingOperationResponse, AccountingResponseStatus, AuthorizationArg, AuthorizationKey, AuthorizationOperation, AuthorizationOperationResponse, diff --git a/libraries/tacacsrs_agent_ipc_emulator/Cargo.toml b/libraries/tacacsrs_agent_ipc_emulator/Cargo.toml index 9ff2a0c7..0aa268d0 100644 --- a/libraries/tacacsrs_agent_ipc_emulator/Cargo.toml +++ b/libraries/tacacsrs_agent_ipc_emulator/Cargo.toml @@ -14,7 +14,7 @@ http = "1.4.1" hyper-util = "0.1.20" log = "0.4" prost = "0.14.3" -regorus = { version = "0.10.1", default-features = false, features = ["arc", "regex"] } +regorus = { version = "0.11.0", default-features = false, features = ["arc", "regex"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tacacsrs-agent-client = { path = "../tacacsrs_agent_client" } diff --git a/libraries/tacacsrs_cli_datastore/Cargo.toml b/libraries/tacacsrs_cli_datastore/Cargo.toml index 0a04e6a9..775470df 100644 --- a/libraries/tacacsrs_cli_datastore/Cargo.toml +++ b/libraries/tacacsrs_cli_datastore/Cargo.toml @@ -9,7 +9,7 @@ repository.workspace = true [dependencies] anyhow = "1" async-trait = "0.1" -base64 = "0.22" +base64 = "0.23" futures-core = "0.3" futures-util = "0.3" log = "0.4" diff --git a/libraries/tacacsrs_cli_datastore/src/datastore.rs b/libraries/tacacsrs_cli_datastore/src/datastore.rs index bf6fd8b7..ccde0336 100644 --- a/libraries/tacacsrs_cli_datastore/src/datastore.rs +++ b/libraries/tacacsrs_cli_datastore/src/datastore.rs @@ -6,8 +6,12 @@ use anyhow::Context; use async_trait::async_trait; use notify::{Config, Event, RecommendedWatcher, RecursiveMode, Watcher}; use tacacsrs_config::TacacsPlus; -use tacacsrs_datastore::{watch_to_change_stream, ConfigChangeStream, ConfigDatastore}; -use tokio::sync::{mpsc, watch}; +use tacacsrs_datastore::{ + ChangeNotificationMode, ConfigChange, ConfigChangeEvent, ConfigChangeStream, ConfigDatastore, + ConfigDelta, DatastoreRuntimePolicy, InitialLoadPolicy, +}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; use crate::builder::tacacs_plus_from_cli_input; use crate::model::CliDatastoreInput; @@ -39,6 +43,16 @@ impl CliFileDatastore { #[async_trait] impl ConfigDatastore for CliFileDatastore { + fn runtime_policy(&self) -> DatastoreRuntimePolicy { + let change_notifications = if self.input.watched_paths().is_empty() { + ChangeNotificationMode::None + } else { + ChangeNotificationMode::Continuous + }; + + DatastoreRuntimePolicy::new(InitialLoadPolicy::FailFast, change_notifications) + } + async fn load(&self) -> anyhow::Result { tacacs_plus_from_cli_input(&self.input) } @@ -50,7 +64,7 @@ impl ConfigDatastore for CliFileDatastore { } let initial = self.load().await.ok().map(Arc::new); - let (snapshot_tx, snapshot_rx) = watch::channel(initial); + let (change_tx, change_rx) = mpsc::channel(8); let (event_tx, mut event_rx) = mpsc::channel(32); let watch_dirs = watch_directories(&watched_paths); let mut watcher = RecommendedWatcher::new( @@ -72,6 +86,7 @@ impl ConfigDatastore for CliFileDatastore { let datastore = self.clone(); tokio::spawn(async move { let _watcher = watcher; + let mut previous = initial; while let Some(event) = event_rx.recv().await { match event { Ok(event) if event_touches_watched_path(&event, &watched_paths) => { @@ -79,7 +94,17 @@ impl ConfigDatastore for CliFileDatastore { while let Ok(Ok(_)) = event_rx.try_recv() {} match datastore.load().await { Ok(snapshot) => { - if snapshot_tx.send(Some(Arc::new(snapshot))).is_err() { + let snapshot = Arc::new(snapshot); + let change = ConfigChange { + delta: ConfigDelta::diff(previous.as_deref(), &snapshot), + config: Arc::clone(&snapshot), + }; + previous = Some(snapshot); + if change_tx + .send(ConfigChangeEvent::Changed(change)) + .await + .is_err() + { log::debug!("CLI file datastore subscriber dropped; exiting"); break; } @@ -88,6 +113,13 @@ impl ConfigDatastore for CliFileDatastore { log::error!( "CLI file datastore reload failed; keeping previous configuration: {err:#}" ); + if change_tx + .send(ConfigChangeEvent::CandidateRejected) + .await + .is_err() + { + break; + } } } } @@ -99,7 +131,7 @@ impl ConfigDatastore for CliFileDatastore { } }); - Ok(watch_to_change_stream(snapshot_rx)) + Ok(Box::pin(ReceiverStream::new(change_rx))) } fn label(&self) -> &'static str { @@ -201,12 +233,16 @@ mod tests { async fn next_change(stream: &mut S) -> tacacsrs_datastore::ConfigChange where - S: Stream + Unpin, + S: Stream + Unpin, { - timeout(Duration::from_secs(5), futures_util::StreamExt::next(stream)) + let event = timeout(Duration::from_secs(5), futures_util::StreamExt::next(stream)) .await .expect("change should arrive") - .expect("stream should yield change") + .expect("stream should yield change"); + let tacacsrs_datastore::ConfigChangeEvent::Changed(change) = event else { + panic!("expected a changed event"); + }; + change } #[tokio::test] @@ -226,6 +262,55 @@ mod tests { assert_eq!(change.config.server[0].address, "192.0.2.11"); } + #[tokio::test] + async fn subscribe_reports_rejected_candidate_after_invalid_file_update() { + let path = temp_config(&config("192.0.2.10")); + let input = + CliDatastoreInput::new(CliConfigSource::YangFile { path: path.clone() }, "file") + .with_debounce(Duration::from_millis(50)); + let datastore = CliFileDatastore::new(input); + let mut stream = datastore.subscribe().await.expect("subscribe should work"); + + fs::write(&path, "not valid JSON").expect("config should update"); + + let event = timeout(Duration::from_secs(5), futures_util::StreamExt::next(&mut stream)) + .await + .expect("rejection should arrive") + .expect("stream should yield rejection"); + fs::remove_file(path).ok(); + + assert!(matches!(event, ConfigChangeEvent::CandidateRejected)); + } + + #[test] + fn runtime_policy_is_continuous_only_when_files_are_watched() { + let inline = CliFileDatastore::new(CliDatastoreInput::new( + CliConfigSource::Inline { + servers: vec![CliServerInput::new("server-0", "192.0.2.10:49")], + security: CliSecurity::from_cli_inputs(CliSecurityInputs::default()), + }, + "cli", + )); + assert_eq!( + inline.runtime_policy(), + DatastoreRuntimePolicy::new(InitialLoadPolicy::FailFast, ChangeNotificationMode::None,) + ); + + let path = temp_config(&config("192.0.2.10")); + let file = CliFileDatastore::new(CliDatastoreInput::new( + CliConfigSource::YangFile { path: path.clone() }, + "file", + )); + assert_eq!( + file.runtime_policy(), + DatastoreRuntimePolicy::new( + InitialLoadPolicy::FailFast, + ChangeNotificationMode::Continuous, + ) + ); + fs::remove_file(path).ok(); + } + #[tokio::test] async fn subscribe_emits_change_after_certificate_file_update() { let cert_path = temp_file("cert", b"cert-a"); diff --git a/libraries/tacacsrs_config/Cargo.toml b/libraries/tacacsrs_config/Cargo.toml index 7a3dff79..2ecece47 100644 --- a/libraries/tacacsrs_config/Cargo.toml +++ b/libraries/tacacsrs_config/Cargo.toml @@ -8,7 +8,7 @@ repository.workspace = true [dependencies] anyhow = "1" -base64 = "0.22" +base64 = "0.23" bitflags = { version = "2", features = ["serde"] } serde = { version = "1", features = ["derive"] } serde_ignored = "0.1" diff --git a/libraries/tacacsrs_config/src/builders.rs b/libraries/tacacsrs_config/src/builders.rs index dbc437d5..45b56fa8 100644 --- a/libraries/tacacsrs_config/src/builders.rs +++ b/libraries/tacacsrs_config/src/builders.rs @@ -11,6 +11,23 @@ pub const DEFAULT_PSK_DHE_KE_GROUPS: &[PskDheKeSupportedGroup] = &[ PskDheKeSupportedGroup::Secp256r1, ]; +impl TacacsPlus { + /// Creates an empty TACACS+ root with no configured servers or credentials. + /// + /// This represents a runtime that is waiting for its first external + /// configuration snapshot. Operator-authored static configurations should + /// continue to use [`TacacsPlusBuilder::build`] so required-server and + /// credential validation is applied. + #[must_use] + pub const fn empty() -> Self { + Self { + client_credentials: Vec::new(), + server_credentials: Vec::new(), + server: Vec::new(), + } + } +} + /// Builder for constructing a [`TacacsPlus`] root configuration in code. /// /// This builder is the in-process counterpart to parsing YANG JSON via @@ -38,11 +55,7 @@ impl TacacsPlusBuilder { #[must_use] pub fn new() -> Self { Self { - root: TacacsPlus { - client_credentials: Vec::new(), - server_credentials: Vec::new(), - server: Vec::new(), - }, + root: TacacsPlus::empty(), } } diff --git a/libraries/tacacsrs_datastore/src/lib.rs b/libraries/tacacsrs_datastore/src/lib.rs index fcc6a1e9..ea9c5514 100644 --- a/libraries/tacacsrs_datastore/src/lib.rs +++ b/libraries/tacacsrs_datastore/src/lib.rs @@ -128,8 +128,63 @@ pub struct ConfigChange { pub delta: ConfigDelta, } -/// Stream of [`ConfigChange`] events returned by [`ConfigDatastore::subscribe`]. -pub type ConfigChangeStream = Pin + Send + 'static>>; +/// Typed event emitted by a datastore change subscription. +#[derive(Debug, Clone)] +pub enum ConfigChangeEvent { + /// A complete validated candidate snapshot is available. + Changed(ConfigChange), + /// The datastore observed a change but rejected the resulting candidate. + /// + /// Backends log their detailed error locally. The event intentionally + /// carries no error text or configuration value so health consumers cannot + /// expose addresses, credential references, or secrets. + CandidateRejected, +} + +/// Stream of [`ConfigChangeEvent`] values returned by [`ConfigDatastore::subscribe`]. +pub type ConfigChangeStream = Pin + Send + 'static>>; + +/// Behavior the runtime applies when the initial datastore load fails. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum InitialLoadPolicy { + /// Return the load error to the operator and stop startup. + FailFast, + /// Retry until a valid snapshot is available or shutdown is requested. + RetryUntilAvailable, +} + +/// Change-notification guarantees provided by a datastore instance. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ChangeNotificationMode { + /// The datastore is immutable for the lifetime of the process. + None, + /// The datastore maintains a continuous subscription that must be restored + /// if setup fails or its stream ends. + Continuous, +} + +/// Runtime behavior declared by a [`ConfigDatastore`] instance. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct DatastoreRuntimePolicy { + /// Initial-load failure behavior. + pub initial_load: InitialLoadPolicy, + /// Change-notification behavior. + pub change_notifications: ChangeNotificationMode, +} + +impl DatastoreRuntimePolicy { + /// Creates a datastore runtime policy. + #[must_use] + pub const fn new( + initial_load: InitialLoadPolicy, + change_notifications: ChangeNotificationMode, + ) -> Self { + Self { + initial_load, + change_notifications, + } + } +} /// Source of TACACS+ configuration for the agent runtime. /// @@ -143,6 +198,13 @@ pub type ConfigChangeStream = Pin + Send + ' /// [`subscribe`]: ConfigDatastore::subscribe #[async_trait] pub trait ConfigDatastore: Send + Sync + 'static { + /// Declares how the runtime should supervise this datastore instance. + /// + /// This is an instance-level contract because a file-backed datastore may + /// be immutable when no referenced paths exist and continuously watched + /// when its effective configuration depends on files. + fn runtime_policy(&self) -> DatastoreRuntimePolicy; + /// Load the current configuration snapshot. /// /// Called once at startup and may be called again by callers that wish to @@ -211,6 +273,10 @@ impl StaticDatastore { #[async_trait] impl ConfigDatastore for StaticDatastore { + fn runtime_policy(&self) -> DatastoreRuntimePolicy { + DatastoreRuntimePolicy::new(InitialLoadPolicy::FailFast, ChangeNotificationMode::None) + } + async fn load(&self) -> anyhow::Result { Ok((*self.config).clone()) } @@ -244,7 +310,7 @@ pub fn watch_to_change_stream( delta, }; previous = Some(snapshot); - Some(change) + Some(ConfigChangeEvent::Changed(change)) }); Box::pin(stream) } @@ -273,6 +339,10 @@ mod tests { let loaded = store.load().await.expect("load should succeed"); assert_eq!(loaded.server.len(), 1); assert_eq!(store.label(), "static"); + assert_eq!( + store.runtime_policy(), + DatastoreRuntimePolicy::new(InitialLoadPolicy::FailFast, ChangeNotificationMode::None,) + ); let mut stream = store.subscribe().await.expect("subscribe should succeed"); assert!(stream.next().await.is_none(), "static stream should be empty"); @@ -333,7 +403,11 @@ mod tests { tx.send(Some(Arc::clone(&updated_arc))) .expect("send update"); - let change = stream.next().await.expect("should receive change"); + let ConfigChangeEvent::Changed(change) = + stream.next().await.expect("should receive change") + else { + panic!("expected a changed event"); + }; assert_eq!(change.delta.modified_servers, vec!["primary"]); assert_eq!(change.config.server[0].timeout, 7); } @@ -348,10 +422,13 @@ mod tests { updated.server[0].timeout = 9; tx.send(Some(Arc::new(updated))).expect("send update"); - let change = stream + let ConfigChangeEvent::Changed(change) = stream .next() .await - .expect("should receive first real update"); + .expect("should receive first real update") + else { + panic!("expected a changed event"); + }; assert_eq!(change.delta.modified_servers, vec!["primary"]); assert_eq!(change.config.server[0].timeout, 9); } diff --git a/libraries/tacacsrs_sonic/examples/configdb_watch.rs b/libraries/tacacsrs_sonic/examples/configdb_watch.rs index 3ba23413..66146635 100644 --- a/libraries/tacacsrs_sonic/examples/configdb_watch.rs +++ b/libraries/tacacsrs_sonic/examples/configdb_watch.rs @@ -64,14 +64,22 @@ async fn main() -> anyhow::Result<()> { .context("subscribe to SONiC ConfigDB changes")?; let mut observed = 0usize; - while let Some(change) = changes.next().await { + while let Some(event) = changes.next().await { observed += 1; println!("change_event: {observed}"); - println!(" added_servers: {:?}", change.delta.added_servers); - println!(" removed_servers: {:?}", change.delta.removed_servers); - println!(" modified_servers: {:?}", change.delta.modified_servers); - println!(" root_metadata_changed: {}", change.delta.root_metadata_changed); - print_snapshot("current", &change.config); + match event { + tacacsrs_datastore::ConfigChangeEvent::Changed(change) => { + println!(" result: changed"); + println!(" added_servers: {:?}", change.delta.added_servers); + println!(" removed_servers: {:?}", change.delta.removed_servers); + println!(" modified_servers: {:?}", change.delta.modified_servers); + println!(" root_metadata_changed: {}", change.delta.root_metadata_changed); + print_snapshot("current", &change.config); + } + tacacsrs_datastore::ConfigChangeEvent::CandidateRejected => { + println!(" result: candidate_rejected"); + } + } if cli .max_events diff --git a/libraries/tacacsrs_sonic/src/lib.rs b/libraries/tacacsrs_sonic/src/lib.rs index 892276d7..5ff4a46d 100644 --- a/libraries/tacacsrs_sonic/src/lib.rs +++ b/libraries/tacacsrs_sonic/src/lib.rs @@ -10,8 +10,12 @@ use std::time::Duration; use anyhow::Context; use async_trait::async_trait; use tacacsrs_config::TacacsPlus; -use tacacsrs_datastore::{watch_to_change_stream, ConfigChangeStream, ConfigDatastore}; -use tokio::sync::watch; +use tacacsrs_datastore::{ + ChangeNotificationMode, ConfigChange, ConfigChangeEvent, ConfigChangeStream, ConfigDatastore, + ConfigDelta, DatastoreRuntimePolicy, InitialLoadPolicy, +}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; pub use mapping::{map_sonic_tables_to_tacacs_plus, sonic_server_name, SonicHash, SonicTacacsTables}; pub use store::{ @@ -49,6 +53,13 @@ impl SonicConfigDb { #[async_trait] impl ConfigDatastore for SonicConfigDb { + fn runtime_policy(&self) -> DatastoreRuntimePolicy { + DatastoreRuntimePolicy::new( + InitialLoadPolicy::RetryUntilAvailable, + ChangeNotificationMode::Continuous, + ) + } + async fn load(&self) -> anyhow::Result { let mut conn = self .settings @@ -65,16 +76,23 @@ impl ConfigDatastore for SonicConfigDb { async fn subscribe(&self) -> anyhow::Result { let settings = self.settings.clone(); let initial = self.load().await.ok().map(Arc::new); - let (tx, rx) = watch::channel(initial); + let (tx, rx) = mpsc::channel(8); let mut signal = spawn_change_notifier(settings.clone()) .await .context("subscribe to SONiC ConfigDB keyspace notifications")?; tokio::spawn(async move { + let mut previous = initial; while signal.recv().await.is_some() { match reload_with_retry(&settings).await { Ok(snapshot) => { - if tx.send(Some(Arc::new(snapshot))).is_err() { + let snapshot = Arc::new(snapshot); + let change = ConfigChange { + delta: ConfigDelta::diff(previous.as_deref(), &snapshot), + config: Arc::clone(&snapshot), + }; + previous = Some(snapshot); + if tx.send(ConfigChangeEvent::Changed(change)).await.is_err() { log::debug!("SONiC datastore subscriber dropped; exiting"); break; } @@ -83,12 +101,15 @@ impl ConfigDatastore for SonicConfigDb { log::error!( "SONiC ConfigDB reload failed; keeping previous configuration: {err:#}" ); + if tx.send(ConfigChangeEvent::CandidateRejected).await.is_err() { + break; + } } } } }); - Ok(watch_to_change_stream(rx)) + Ok(Box::pin(ReceiverStream::new(rx))) } fn label(&self) -> &'static str { @@ -126,3 +147,21 @@ async fn try_reload(settings: &SonicConnection) -> anyhow::Result { let snapshot = read_tacacs_tables(&mut conn).await?; map_sonic_tables_to_tacacs_plus(&snapshot) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn runtime_policy_retries_and_maintains_continuous_notifications() { + let datastore = SonicConfigDb::new(SonicConnection::default()); + + assert_eq!( + datastore.runtime_policy(), + DatastoreRuntimePolicy::new( + InitialLoadPolicy::RetryUntilAvailable, + ChangeNotificationMode::Continuous, + ) + ); + } +} diff --git a/libraries/tacacsrs_sonic/src/mapping.rs b/libraries/tacacsrs_sonic/src/mapping.rs index 3dab6e52..7ad7d777 100644 --- a/libraries/tacacsrs_sonic/src/mapping.rs +++ b/libraries/tacacsrs_sonic/src/mapping.rs @@ -97,11 +97,7 @@ impl SonicTacacsTables { /// non-empty configuration fails validation. pub fn map_sonic_tables_to_tacacs_plus(tables: &SonicTacacsTables) -> anyhow::Result { if tables.servers.is_empty() { - return Ok(TacacsPlus { - client_credentials: Vec::new(), - server_credentials: Vec::new(), - server: Vec::new(), - }); + return Ok(TacacsPlus::empty()); } let global = SonicGlobal::from_hash(&tables.global)?;