1/*
  2 * SPDX-FileCopyrightText: 2026 Copyright (c) Contributors to the Eclipse Foundation
  3 *
  4 * See the NOTICE file(s) distributed with this work for additional
  5 * information regarding copyright ownership.
  6 *
  7 * This program and the accompanying materials are made available under the
  8 * terms of the Apache License Version 2.0 which is available at
  9 * https://www.apache.org/licenses/LICENSE-2.0
 10 *
 11 * SPDX-License-Identifier: Apache-2.0
 12 */
 13
 14use std::time::Duration;
 15
 16use cda_health::{HealthState, Status as HealthStatus};
 17use cda_interfaces::spawn_named;
 18use tokio::task::JoinHandle;
 19
 20/// Hard lower bound for the watchdog notification interval. `tokio::time::interval`
 21/// panics on a zero-length period, so any computed interval is clamped to this.
 22const MIN_NOTIFY_INTERVAL: Duration = Duration::from_millis(1);
 23
 24/// Lower bound below which a watchdog notification interval is considered a
 25/// misconfiguration rather than an intent. Intervals below this still work, but
 26/// are warned about once at startup.
 27const SANE_MIN_NOTIFY_INTERVAL: Duration = Duration::from_secs(1);
 28
 29/// Creates a background task that integrates with the systemd watchdog.
 30///
[docs] 31/// [[ dimpl~system-sd-notify-watchdog-integration, Systemd Watchdog Integration ]]
 32///
 33/// If the process was booted via systemd and the watchdog is enabled,
 34/// this function spawns a task that periodically checks the health state
 35/// and sends the appropriate `sd_notify` notification:
 36///
 37/// - `Ready` when transitioning from `Starting` to `Up`
 38/// - `Watchdog` while healthy (`Up` -> `Up`)
 39/// - `WatchdogTrigger` when health degrades (`Up` -> `Failed`)
 40///
 41/// The notification interval is half of the systemd-configured watchdog timeout,
 42/// per the systemd-recommended convention.
 43///
 44/// When the shutdown signal fires, `Stopping` is sent so systemd treats the exit
 45/// as intentional rather than a watchdog failure.
 46///
 47/// If systemd is not detected or the watchdog is not enabled, returns `None`
 48/// and the CDA runs without `sd_notify` behavior.
 49pub fn create_sd_notify_task<F>(
 50    health_state: Option<HealthState>,
 51    shutdown_signal: F,
 52) -> Option<JoinHandle<()>>
 53where
 54    F: Future<Output = ()> + Clone + Send + 'static,
 55{
 56    // map std_notify::booted Err(_) to false and treat as not systemd booted
 57    if !sd_notify::booted().unwrap_or(false) {
 58        tracing::info!("Systemd not detected, skipping sd_notify initialization");
 59        return None;
 60    }
 61    let Some(interval) = watchdog_notify_interval() else {
 62        tracing::info!("Systemd watchdog not enabled, skipping sd_notify initialization");
 63        return None;
 64    };
 65    // init sd notify task
 66    let watchdog_future = async move {
 67        let mut interval_timer = tokio::time::interval(interval);
 68        let mut state = cda_health::Status::Starting;
 69        loop {
 70            interval_timer.tick().await;
 71            state = trigger_watchdog(health_state.as_ref(), state).await;
 72        }
 73    };
 74    let sd_notify_future = spawn_named!("sd_notify", async move {
 75        tokio::select! {
 76            _ = watchdog_future => {},
 77            () = shutdown_signal => {
 78                tracing::debug!("Shutdown signal received, notifying systemd of stopping state");
 79                if let Err(e) = sd_notify::notify(&[sd_notify::NotifyState::Stopping]) {
 80                    tracing::warn!(error = %e, "Failed to send sd_notify stopping notification");
 81                }
 82            },
 83        }
 84    });
 85    tracing::info!(
 86        "Systemd detected and watchdog enabled, initialized sd_notify task with interval of \
 87         {interval:?}"
 88    );
 89    Some(sd_notify_future)
 90}
 91
 92/// Notifies systemd that startup has finished, by sending `READY=1`.
 93///
 94/// Services declared as `Type=notify` are not considered started by systemd until
 95/// this notification arrives, regardless of whether the watchdog is enabled. It is
 96/// therefore sent independently of [`create_sd_notify_task`], once the CDA has
 97/// finished loading and is able to serve requests.
 98///
 99/// Does nothing when the process was not booted via systemd.
100pub fn notify_ready() {
101    // map std_notify::booted Err(_) to false and treat as not systemd booted
102    if !sd_notify::booted().unwrap_or(false) {
103        tracing::debug!("Systemd not detected, skipping sd_notify ready notification");
104        return;
105    }
106    tracing::debug!("Notifying systemd that startup has finished");
107    if let Err(e) = sd_notify::notify(&[sd_notify::NotifyState::Ready]) {
108        tracing::warn!(error = %e, "Failed to send sd_notify ready notification");
109    }
110}
111
112/// Interval at which watchdog notifications should be sent: half of the
113/// systemd-configured watchdog timeout, per the systemd-recommended convention
114/// (see `sd_watchdog_enabled(3)`).
115///
116/// Clamped to at least 1ms so a very small (or zero) `WatchdogSec` can never
117/// produce a zero-length interval, which `tokio::time::interval` panics on.
118///
119/// systemd accepts watchdog timeouts down to microsecond resolution, which is a
120/// legal but implausible configuration for the CDA: it would wake the notify task
121/// hundreds or thousands of times per second for no benefit. Such a configuration
122/// is honoured, but warned about once at startup.
123fn watchdog_notify_interval() -> Option<Duration> {
124    sd_notify::watchdog_enabled().map(|timeout| {
125        // the divisor is a non-zero literal, so the fallback can never be taken.
126        let interval = timeout
127            .checked_div(2)
128            .unwrap_or(timeout)
129            .max(MIN_NOTIFY_INTERVAL);
130        if interval < SANE_MIN_NOTIFY_INTERVAL {
131            tracing::warn!(
132                "Systemd watchdog timeout of {timeout:?} results in a notification interval of \
133                 {interval:?}, which will cause considerable system load. Consider configuring a \
134                 WatchdogSec of at least {:?}.",
135                SANE_MIN_NOTIFY_INTERVAL.saturating_mul(2)
136            );
137        }
138        interval
139    })
140}
141
142async fn trigger_watchdog(
143    health_state: Option<&HealthState>,
144    prev_state: cda_health::Status,
145) -> cda_health::Status {
146    let new_status = fold_health_state(health_state).await;
147    let notify = match (prev_state, new_status) {
148        (cda_health::Status::Starting, cda_health::Status::Up) => sd_notify::NotifyState::Ready,
149        (cda_health::Status::Up, cda_health::Status::Failed) => {
150            sd_notify::NotifyState::WatchdogTrigger
151        }
152        (cda_health::Status::Up, cda_health::Status::Up) => sd_notify::NotifyState::Watchdog,
153        // this would be Failure -> Starting or Up -> Starting. so makes no sense
154        _ => {
155            tracing::warn!(
156                "Unexpected health status transition from {prev_state:?} to {new_status:?} when \
157                 triggering sd_notify watchdog"
158            );
159            sd_notify::NotifyState::Watchdog
160        }
161    };
162    tracing::debug!("Triggering sd_notify watchdog with status {notify:?}");
163    if let Err(e) = sd_notify::notify(&[notify]) {
164        tracing::warn!(error = %e, "Failed to send sd_notify watchdog notification");
165    }
166    new_status
167}
168
169fn fold_status(acc: HealthStatus, status: HealthStatus) -> HealthStatus {
170    match (acc, status) {
171        (
172            HealthStatus::Up | HealthStatus::Starting | HealthStatus::Pending,
173            HealthStatus::Failed,
174        )
175        | (HealthStatus::Failed, _) => HealthStatus::Failed,
176        (HealthStatus::Starting | HealthStatus::Pending, HealthStatus::Pending) => {
177            HealthStatus::Starting
178        }
179        (HealthStatus::Starting | HealthStatus::Pending, HealthStatus::Up) => HealthStatus::Up,
180        (_, status) => status,
181    }
182}
183
184async fn fold_health_state(health_state: Option<&HealthState>) -> HealthStatus {
185    let Some(health_state) = health_state else {
186        return cda_health::Status::Up;
187    };
188    health_state
189        .query_all_providers()
190        .await
191        .into_values()
192        .fold(cda_health::Status::Starting, fold_status)
193}
194
[docs]195/// [[ test~system-sd-notify-watchdog-integration, Systemd Watchdog Health Aggregation Tests, test ]]
196#[cfg(test)]
197mod tests {
198    use cda_health::Status;
199
200    use super::fold_status;
201
202    #[test]
203    fn fold_up_then_failed_yields_failed() {
204        assert_eq!(fold_status(Status::Up, Status::Failed), Status::Failed);
205    }
206
207    #[test]
208    fn fold_starting_then_failed_yields_failed() {
209        assert_eq!(
210            fold_status(Status::Starting, Status::Failed),
211            Status::Failed
212        );
213    }
214
215    #[test]
216    fn fold_pending_then_failed_yields_failed() {
217        assert_eq!(fold_status(Status::Pending, Status::Failed), Status::Failed);
218    }
219
220    #[test]
221    fn fold_failed_then_up_yields_failed() {
222        assert_eq!(fold_status(Status::Failed, Status::Up), Status::Failed);
223    }
224
225    #[test]
226    fn fold_failed_then_starting_yields_failed() {
227        assert_eq!(
228            fold_status(Status::Failed, Status::Starting),
229            Status::Failed
230        );
231    }
232
233    #[test]
234    fn fold_failed_then_pending_yields_failed() {
235        assert_eq!(fold_status(Status::Failed, Status::Pending), Status::Failed);
236    }
237
238    #[test]
239    fn fold_failed_then_failed_yields_failed() {
240        assert_eq!(fold_status(Status::Failed, Status::Failed), Status::Failed);
241    }
242
243    #[test]
244    fn fold_starting_then_pending_yields_starting() {
245        assert_eq!(
246            fold_status(Status::Starting, Status::Pending),
247            Status::Starting
248        );
249    }
250
251    #[test]
252    fn fold_pending_then_pending_yields_starting() {
253        assert_eq!(
254            fold_status(Status::Pending, Status::Pending),
255            Status::Starting
256        );
257    }
258
259    #[test]
260    fn fold_starting_then_up_yields_up() {
261        assert_eq!(fold_status(Status::Starting, Status::Up), Status::Up);
262    }
263
264    #[test]
265    fn fold_pending_then_up_yields_up() {
266        assert_eq!(fold_status(Status::Pending, Status::Up), Status::Up);
267    }
268
269    #[test]
270    fn fold_up_then_up_yields_up() {
271        assert_eq!(fold_status(Status::Up, Status::Up), Status::Up);
272    }
273
274    #[test]
275    fn fold_up_then_starting_yields_starting() {
276        assert_eq!(fold_status(Status::Up, Status::Starting), Status::Starting);
277    }
278
279    #[test]
280    fn fold_up_then_pending_yields_pending() {
281        assert_eq!(fold_status(Status::Up, Status::Pending), Status::Pending);
282    }
283
284    #[test]
285    fn fold_sequence_all_up_from_starting() {
286        // initial acc = Starting (as fold_health_state does), all providers Up
287        let result = [Status::Up, Status::Up, Status::Up]
288            .into_iter()
289            .fold(Status::Starting, fold_status);
290        assert_eq!(result, Status::Up);
291    }
292
293    #[test]
294    fn fold_sequence_any_failed_dominates() {
295        let result = [Status::Up, Status::Failed, Status::Up]
296            .into_iter()
297            .fold(Status::Starting, fold_status);
298        assert_eq!(result, Status::Failed);
299    }
300
301    #[test]
302    fn fold_sequence_failed_at_start_stays_failed() {
303        let result = [Status::Failed, Status::Up, Status::Up]
304            .into_iter()
305            .fold(Status::Starting, fold_status);
306        assert_eq!(result, Status::Failed);
307    }
308
309    #[test]
310    fn fold_sequence_all_starting_stays_starting() {
311        let result = [Status::Starting, Status::Starting]
312            .into_iter()
313            .fold(Status::Starting, fold_status);
314        assert_eq!(result, Status::Starting);
315    }
316
317    #[test]
318    fn fold_sequence_mix_starting_pending_yields_starting() {
319        let result = [Status::Pending, Status::Starting, Status::Pending]
320            .into_iter()
321            .fold(Status::Starting, fold_status);
322        assert_eq!(result, Status::Starting);
323    }
324
325    #[test]
326    fn fold_sequence_empty_providers_yields_starting() {
327        // no providers -> fold over empty iterator -> initial accumulator = Starting
328        let result = std::iter::empty::<Status>().fold(Status::Starting, fold_status);
329        assert_eq!(result, Status::Starting);
330    }
331
332    #[tokio::test]
333    async fn fold_health_state_none_yields_up() {
334        let result = super::fold_health_state(None).await;
335        assert_eq!(result, Status::Up);
336    }
337}