1/*
2 * SPDX-FileCopyrightText: 2026 Copyright (c) Contributors to the Eclipse Foundation
3 *
4 * See the NOTICE file(s) distributed with this work for additional
5 * information regarding copyright ownership.
6 *
7 * This program and the accompanying materials are made available under the
8 * terms of the Apache License Version 2.0 which is available at
9 * https://www.apache.org/licenses/LICENSE-2.0
10 *
11 * SPDX-License-Identifier: Apache-2.0
12 */
13
14use std::time::Duration;
15
16use cda_health::{HealthState, Status as HealthStatus};
17use cda_interfaces::spawn_named;
18use tokio::task::JoinHandle;
19
20/// Hard lower bound for the watchdog notification interval. `tokio::time::interval`
21/// panics on a zero-length period, so any computed interval is clamped to this.
22const MIN_NOTIFY_INTERVAL: Duration = Duration::from_millis(1);
23
24/// Lower bound below which a watchdog notification interval is considered a
25/// misconfiguration rather than an intent. Intervals below this still work, but
26/// are warned about once at startup.
27const SANE_MIN_NOTIFY_INTERVAL: Duration = Duration::from_secs(1);
28
29/// Creates a background task that integrates with the systemd watchdog.
30///
[docs] 31/// [[ dimpl~system-sd-notify-watchdog-integration, Systemd Watchdog Integration ]]
32///
33/// If the process was booted via systemd and the watchdog is enabled,
34/// this function spawns a task that periodically checks the health state
35/// and sends the appropriate `sd_notify` notification:
36///
37/// - `Ready` when transitioning from `Starting` to `Up`
38/// - `Watchdog` while healthy (`Up` -> `Up`)
39/// - `WatchdogTrigger` when health degrades (`Up` -> `Failed`)
40///
41/// The notification interval is half of the systemd-configured watchdog timeout,
42/// per the systemd-recommended convention.
43///
44/// When the shutdown signal fires, `Stopping` is sent so systemd treats the exit
45/// as intentional rather than a watchdog failure.
46///
47/// If systemd is not detected or the watchdog is not enabled, returns `None`
48/// and the CDA runs without `sd_notify` behavior.
49pub fn create_sd_notify_task<F>(
50 health_state: Option<HealthState>,
51 shutdown_signal: F,
52) -> Option<JoinHandle<()>>
53where
54 F: Future<Output = ()> + Clone + Send + 'static,
55{
56 // map std_notify::booted Err(_) to false and treat as not systemd booted
57 if !sd_notify::booted().unwrap_or(false) {
58 tracing::info!("Systemd not detected, skipping sd_notify initialization");
59 return None;
60 }
61 let Some(interval) = watchdog_notify_interval() else {
62 tracing::info!("Systemd watchdog not enabled, skipping sd_notify initialization");
63 return None;
64 };
65 // init sd notify task
66 let watchdog_future = async move {
67 let mut interval_timer = tokio::time::interval(interval);
68 let mut state = cda_health::Status::Starting;
69 loop {
70 interval_timer.tick().await;
71 state = trigger_watchdog(health_state.as_ref(), state).await;
72 }
73 };
74 let sd_notify_future = spawn_named!("sd_notify", async move {
75 tokio::select! {
76 _ = watchdog_future => {},
77 () = shutdown_signal => {
78 tracing::debug!("Shutdown signal received, notifying systemd of stopping state");
79 if let Err(e) = sd_notify::notify(&[sd_notify::NotifyState::Stopping]) {
80 tracing::warn!(error = %e, "Failed to send sd_notify stopping notification");
81 }
82 },
83 }
84 });
85 tracing::info!(
86 "Systemd detected and watchdog enabled, initialized sd_notify task with interval of \
87 {interval:?}"
88 );
89 Some(sd_notify_future)
90}
91
92/// Notifies systemd that startup has finished, by sending `READY=1`.
93///
94/// Services declared as `Type=notify` are not considered started by systemd until
95/// this notification arrives, regardless of whether the watchdog is enabled. It is
96/// therefore sent independently of [`create_sd_notify_task`], once the CDA has
97/// finished loading and is able to serve requests.
98///
99/// Does nothing when the process was not booted via systemd.
100pub fn notify_ready() {
101 // map std_notify::booted Err(_) to false and treat as not systemd booted
102 if !sd_notify::booted().unwrap_or(false) {
103 tracing::debug!("Systemd not detected, skipping sd_notify ready notification");
104 return;
105 }
106 tracing::debug!("Notifying systemd that startup has finished");
107 if let Err(e) = sd_notify::notify(&[sd_notify::NotifyState::Ready]) {
108 tracing::warn!(error = %e, "Failed to send sd_notify ready notification");
109 }
110}
111
112/// Interval at which watchdog notifications should be sent: half of the
113/// systemd-configured watchdog timeout, per the systemd-recommended convention
114/// (see `sd_watchdog_enabled(3)`).
115///
116/// Clamped to at least 1ms so a very small (or zero) `WatchdogSec` can never
117/// produce a zero-length interval, which `tokio::time::interval` panics on.
118///
119/// systemd accepts watchdog timeouts down to microsecond resolution, which is a
120/// legal but implausible configuration for the CDA: it would wake the notify task
121/// hundreds or thousands of times per second for no benefit. Such a configuration
122/// is honoured, but warned about once at startup.
123fn watchdog_notify_interval() -> Option<Duration> {
124 sd_notify::watchdog_enabled().map(|timeout| {
125 // the divisor is a non-zero literal, so the fallback can never be taken.
126 let interval = timeout
127 .checked_div(2)
128 .unwrap_or(timeout)
129 .max(MIN_NOTIFY_INTERVAL);
130 if interval < SANE_MIN_NOTIFY_INTERVAL {
131 tracing::warn!(
132 "Systemd watchdog timeout of {timeout:?} results in a notification interval of \
133 {interval:?}, which will cause considerable system load. Consider configuring a \
134 WatchdogSec of at least {:?}.",
135 SANE_MIN_NOTIFY_INTERVAL.saturating_mul(2)
136 );
137 }
138 interval
139 })
140}
141
142async fn trigger_watchdog(
143 health_state: Option<&HealthState>,
144 prev_state: cda_health::Status,
145) -> cda_health::Status {
146 let new_status = fold_health_state(health_state).await;
147 let notify = match (prev_state, new_status) {
148 (cda_health::Status::Starting, cda_health::Status::Up) => sd_notify::NotifyState::Ready,
149 (cda_health::Status::Up, cda_health::Status::Failed) => {
150 sd_notify::NotifyState::WatchdogTrigger
151 }
152 (cda_health::Status::Up, cda_health::Status::Up) => sd_notify::NotifyState::Watchdog,
153 // this would be Failure -> Starting or Up -> Starting. so makes no sense
154 _ => {
155 tracing::warn!(
156 "Unexpected health status transition from {prev_state:?} to {new_status:?} when \
157 triggering sd_notify watchdog"
158 );
159 sd_notify::NotifyState::Watchdog
160 }
161 };
162 tracing::debug!("Triggering sd_notify watchdog with status {notify:?}");
163 if let Err(e) = sd_notify::notify(&[notify]) {
164 tracing::warn!(error = %e, "Failed to send sd_notify watchdog notification");
165 }
166 new_status
167}
168
169fn fold_status(acc: HealthStatus, status: HealthStatus) -> HealthStatus {
170 match (acc, status) {
171 (
172 HealthStatus::Up | HealthStatus::Starting | HealthStatus::Pending,
173 HealthStatus::Failed,
174 )
175 | (HealthStatus::Failed, _) => HealthStatus::Failed,
176 (HealthStatus::Starting | HealthStatus::Pending, HealthStatus::Pending) => {
177 HealthStatus::Starting
178 }
179 (HealthStatus::Starting | HealthStatus::Pending, HealthStatus::Up) => HealthStatus::Up,
180 (_, status) => status,
181 }
182}
183
184async fn fold_health_state(health_state: Option<&HealthState>) -> HealthStatus {
185 let Some(health_state) = health_state else {
186 return cda_health::Status::Up;
187 };
188 health_state
189 .query_all_providers()
190 .await
191 .into_values()
192 .fold(cda_health::Status::Starting, fold_status)
193}
194
[docs]195/// [[ test~system-sd-notify-watchdog-integration, Systemd Watchdog Health Aggregation Tests, test ]]
196#[cfg(test)]
197mod tests {
198 use cda_health::Status;
199
200 use super::fold_status;
201
202 #[test]
203 fn fold_up_then_failed_yields_failed() {
204 assert_eq!(fold_status(Status::Up, Status::Failed), Status::Failed);
205 }
206
207 #[test]
208 fn fold_starting_then_failed_yields_failed() {
209 assert_eq!(
210 fold_status(Status::Starting, Status::Failed),
211 Status::Failed
212 );
213 }
214
215 #[test]
216 fn fold_pending_then_failed_yields_failed() {
217 assert_eq!(fold_status(Status::Pending, Status::Failed), Status::Failed);
218 }
219
220 #[test]
221 fn fold_failed_then_up_yields_failed() {
222 assert_eq!(fold_status(Status::Failed, Status::Up), Status::Failed);
223 }
224
225 #[test]
226 fn fold_failed_then_starting_yields_failed() {
227 assert_eq!(
228 fold_status(Status::Failed, Status::Starting),
229 Status::Failed
230 );
231 }
232
233 #[test]
234 fn fold_failed_then_pending_yields_failed() {
235 assert_eq!(fold_status(Status::Failed, Status::Pending), Status::Failed);
236 }
237
238 #[test]
239 fn fold_failed_then_failed_yields_failed() {
240 assert_eq!(fold_status(Status::Failed, Status::Failed), Status::Failed);
241 }
242
243 #[test]
244 fn fold_starting_then_pending_yields_starting() {
245 assert_eq!(
246 fold_status(Status::Starting, Status::Pending),
247 Status::Starting
248 );
249 }
250
251 #[test]
252 fn fold_pending_then_pending_yields_starting() {
253 assert_eq!(
254 fold_status(Status::Pending, Status::Pending),
255 Status::Starting
256 );
257 }
258
259 #[test]
260 fn fold_starting_then_up_yields_up() {
261 assert_eq!(fold_status(Status::Starting, Status::Up), Status::Up);
262 }
263
264 #[test]
265 fn fold_pending_then_up_yields_up() {
266 assert_eq!(fold_status(Status::Pending, Status::Up), Status::Up);
267 }
268
269 #[test]
270 fn fold_up_then_up_yields_up() {
271 assert_eq!(fold_status(Status::Up, Status::Up), Status::Up);
272 }
273
274 #[test]
275 fn fold_up_then_starting_yields_starting() {
276 assert_eq!(fold_status(Status::Up, Status::Starting), Status::Starting);
277 }
278
279 #[test]
280 fn fold_up_then_pending_yields_pending() {
281 assert_eq!(fold_status(Status::Up, Status::Pending), Status::Pending);
282 }
283
284 #[test]
285 fn fold_sequence_all_up_from_starting() {
286 // initial acc = Starting (as fold_health_state does), all providers Up
287 let result = [Status::Up, Status::Up, Status::Up]
288 .into_iter()
289 .fold(Status::Starting, fold_status);
290 assert_eq!(result, Status::Up);
291 }
292
293 #[test]
294 fn fold_sequence_any_failed_dominates() {
295 let result = [Status::Up, Status::Failed, Status::Up]
296 .into_iter()
297 .fold(Status::Starting, fold_status);
298 assert_eq!(result, Status::Failed);
299 }
300
301 #[test]
302 fn fold_sequence_failed_at_start_stays_failed() {
303 let result = [Status::Failed, Status::Up, Status::Up]
304 .into_iter()
305 .fold(Status::Starting, fold_status);
306 assert_eq!(result, Status::Failed);
307 }
308
309 #[test]
310 fn fold_sequence_all_starting_stays_starting() {
311 let result = [Status::Starting, Status::Starting]
312 .into_iter()
313 .fold(Status::Starting, fold_status);
314 assert_eq!(result, Status::Starting);
315 }
316
317 #[test]
318 fn fold_sequence_mix_starting_pending_yields_starting() {
319 let result = [Status::Pending, Status::Starting, Status::Pending]
320 .into_iter()
321 .fold(Status::Starting, fold_status);
322 assert_eq!(result, Status::Starting);
323 }
324
325 #[test]
326 fn fold_sequence_empty_providers_yields_starting() {
327 // no providers -> fold over empty iterator -> initial accumulator = Starting
328 let result = std::iter::empty::<Status>().fold(Status::Starting, fold_status);
329 assert_eq!(result, Status::Starting);
330 }
331
332 #[tokio::test]
333 async fn fold_health_state_none_yields_up() {
334 let result = super::fold_health_state(None).await;
335 assert_eq!(result, Status::Up);
336 }
337}