1/*
2 * SPDX-FileCopyrightText: 2026 Copyright (c) Contributors to the Eclipse Foundation
3 *
4 * See the NOTICE file(s) distributed with this work for additional
5 * information regarding copyright ownership.
6 *
7 * This program and the accompanying materials are made available under the
8 * terms of the Apache License Version 2.0 which is available at
9 * https://www.apache.org/licenses/LICENSE-2.0
10 *
11 * SPDX-License-Identifier: Apache-2.0
12 */
13
14//! [`CommunicationHandle`] claims lifecycle-state transitions synchronously and
15//! submits the resulting work to the `worker`, which owns the transport and the
16//! initializer list.
17
18use std::{fmt::Display, sync::Arc};
19
20use cda_interfaces::{
21 communication_control::{
22 CommunicationError, CommunicationGuard, CommunicationInitMode, CommunicationLifecycle,
23 CommunicationOperation, CommunicationOperationFailure, CommunicationState,
24 CommunicationVariantDetection, TransportControl, VariantDetectionMode,
25 },
26 dlt_ctx,
27};
28use tokio::{
29 sync::{oneshot, watch},
30 task::JoinHandle,
31};
32
33use super::{
34 disable::{DisableError, DisableLease, DisableLeaseId, DisableOwner, DisableReason},
35 guard::ActiveGuard,
36 state::{CommunicationStateStore, EnablingResultReceiver, detection_mode},
37 transition::{self, LifecycleDecision, LifecycleRequest},
38 worker::{self, LifecycleCommand, WorkerSender},
39};
40use crate::plugin::{CommunicationPlugin, CommunicationPluginBuilder};
41
42/// Test helpers for downstream plugin implementors. [`CommunicationHandle`]'s
43/// constructor is private, so production code goes through
44/// `build_communication_runtime`.
45#[cfg(any(test, feature = "test-utils"))]
46pub mod test_utils;
47
48/// Result of applying a [`LifecycleDecision`].
49enum ClaimOutcome {
50 /// The operation already settled. Nothing to await.
51 Settled(Result<CommunicationState, CommunicationOperationFailure>),
52 /// An exclusive disable lease owns the state, which is either
53 /// [`CommunicationState::Disabling`] or
54 /// [`CommunicationState::DisabledExclusive`].
55 LeaseHeld(CommunicationState),
56 /// An operation was claimed or joined. Await the receiver for the result.
57 ///
58 /// The receiver also identifies the attempt, via
59 /// [`watch::Receiver::same_channel`].
60 Pending(EnablingResultReceiver),
61}
62
63/// The capability a [`CommunicationPlugin`] uses to perform lifecycle
64/// operations. Passed to [`CommunicationPluginBuilder::build`]. Plugins
65/// typically retain it and delegate their operations to it.
[docs] 66/// [[ dimpl~communication-lifecycle-controller, Serialized communication lifecycle controller, dimpl ]]
67#[derive(Clone)]
68pub struct CommunicationHandle {
69 state: Arc<CommunicationStateStore>,
70 worker: WorkerSender,
71 worker_task: Arc<tokio::sync::Mutex<Option<JoinHandle<()>>>>,
72}
73
74/// Failure while constructing the authoritative communication plugin runtime.
75#[derive(thiserror::Error, Debug)]
76pub enum BuildCommunicationRuntimeError<E: Display> {
77 /// The selected plugin builder failed.
78 #[error("Communication plugin construction failed: {0}")]
79 Plugin(E),
80}
81
82impl CommunicationHandle {
83 /// Returns the current authoritative communication lifecycle state.
84 #[must_use]
85 pub fn state(&self) -> CommunicationState {
86 self.state.lock().state.clone()
87 }
88
89 /// Acquires diagnostic communication while it is enabled.
90 ///
91 /// Holding a guard refuses a `disable()` and a new whole-vehicle detection.
92 /// A detection already in flight does not refuse guards.
93 ///
94 /// # Errors
95 ///
96 /// Returns an error when communication is not enabled.
97 pub fn acquire(&self) -> Result<CommunicationGuard, CommunicationError> {
98 let mut state = self.state.lock();
99 transition::guard_admission(&state.state)?;
100 // Must be atomic with the admission check above. Otherwise a
101 // `disable()` could claim `Enabled -> Disabling` in the gap.
102 Ok(ActiveGuard::register(&mut state, Arc::clone(&self.state)))
103 }
104
105 /// Registers a hook that runs during subsequent lifecycle transitions.
106 ///
107 /// Hooks run in registration order during activation and detection, and in
108 /// reverse order during deinitialization. Registering the same hook twice
109 /// runs it twice.
110 ///
111 /// # Errors
112 ///
113 /// Returns an error when the lifecycle worker is shutting down or unavailable.
114 pub async fn register_lifecycle_hook(
115 &self,
116 initializer: Arc<dyn CommunicationLifecycle>,
117 ) -> Result<(), CommunicationOperationFailure> {
118 self.submit_and_await(CommunicationOperation::RegisterLifecycleHook, |reply| {
119 LifecycleCommand::RegisterLifecycleHook { initializer, reply }
120 })
121 .await
122 }
123
124 /// Installs the whole-vehicle variant detector.
125 ///
126 /// A single slot. Registering a second detector replaces the first. Until
127 /// one is registered, no operation settles any ECU's variant and
128 /// [`variant_detection`](Self::variant_detection) reports `Never`.
129 ///
130 /// # Errors
131 ///
132 /// Returns an error when the lifecycle worker is shutting down or unavailable.
133 pub async fn register_variant_detection(
134 &self,
135 detector: Arc<dyn CommunicationVariantDetection>,
136 ) -> Result<(), CommunicationOperationFailure> {
137 self.submit_and_await(CommunicationOperation::RegisterVariantDetection, |reply| {
138 LifecycleCommand::RegisterVariantDetection { detector, reply }
139 })
140 .await
141 }
142
143 fn construct_worker_failure(
144 &self,
145 operation: CommunicationOperation,
146 ) -> CommunicationOperationFailure {
147 if self.worker.is_shutting_down() {
148 CommunicationOperationFailure::ShuttingDown { operation }
149 } else {
150 CommunicationOperationFailure::WorkerUnavailable { operation }
151 }
152 }
153
154 /// Sends a worker command built from a fresh reply channel, mapping a send
155 /// failure to [`worker_failure`](Self::construct_worker_failure). Returns the reply
156 /// channel un-awaited, for the callers that handle the receive side
157 /// themselves. See [`submit_and_await`](Self::submit_and_await) for the
158 /// common case.
159 async fn submit<T>(
160 &self,
161 operation: CommunicationOperation,
162 build: impl FnOnce(
163 oneshot::Sender<Result<T, CommunicationOperationFailure>>,
164 ) -> LifecycleCommand,
165 ) -> Result<
166 oneshot::Receiver<Result<T, CommunicationOperationFailure>>,
167 CommunicationOperationFailure,
168 > {
169 let (tx, rx) = oneshot::channel();
170 self.worker
171 .send(build(tx))
172 .await
173 .map_err(|_| self.construct_worker_failure(operation))?;
174 Ok(rx)
175 }
176
177 /// As [`submit`](Self::submit), but awaits the reply, mapping a closed
178 /// channel to [`worker_failure`](Self::construct_worker_failure).
179 async fn submit_and_await<T>(
180 &self,
181 operation: CommunicationOperation,
182 build: impl FnOnce(
183 oneshot::Sender<Result<T, CommunicationOperationFailure>>,
184 ) -> LifecycleCommand,
185 ) -> Result<T, CommunicationOperationFailure> {
186 self.submit(operation, build)
187 .await?
188 .await
189 .unwrap_or_else(|_| Err(self.construct_worker_failure(operation)))
190 }
191
192 /// Activates communication, joining an already in-flight activation if one exists.
193 ///
194 /// Runs all three stages: transport, lifecycle hooks, then variant
195 /// detection. The last is subject to the configured
196 /// [`VariantDetectionMode`]. Use [`enable`](Self::enable) to bring
197 /// communication up without detecting.
198 ///
199 /// Stages run only when this claims a transition. Called while already
200 /// `Enabled`, it returns `Ok(Enabled)` and runs no detector. Use
201 /// [`redetect`](Self::redetect) to detect against a live transport.
202 /// Detection concludes per ECU and asynchronously, so `VariantState` is the
203 /// readiness signal, not this method's return value.
204 ///
205 /// # Errors
206 ///
207 /// Returns an error when the transport, a lifecycle hook, or detection
208 /// fails, an exclusive disable lease currently owns the state, or the
209 /// lifecycle worker is shutting down.
210 #[tracing::instrument(skip_all, fields(dlt_context = dlt_ctx!("COMM")))]
211 pub async fn enable_and_detect(
212 &self,
213 ) -> Result<CommunicationState, CommunicationOperationFailure> {
214 self.run_claimed(LifecycleRequest::EnableAndDetect).await
215 }
216
217 /// Non-awaiting counterpart to
218 /// [`enable_and_detect`](Self::enable_and_detect). Claims the transition or
219 /// joins an in-flight one, then returns immediately and drives the work on a
220 /// detached task.
221 ///
222 /// A returned [`CommunicationState::Enabled`] means there was nothing to
223 /// claim, so no detector ran.
224 ///
225 /// If the detached task panics or is aborted before any caller joins it, the
226 /// state stays `Enabling(_)` until the next caller claims or joins the same
227 /// operation.
228 #[must_use]
229 pub fn request_enable_and_detect(&self) -> CommunicationState {
230 self.request_claimed(LifecycleRequest::EnableAndDetect)
231 }
232
233 /// Enables communication without running variant detection. The transport
234 /// comes up and every registered lifecycle hook initializes, but nothing
235 /// settles any ECU's variant.
236 ///
237 /// Every ECU stays `NotTested` afterwards and
238 /// [`variant_detection`](Self::variant_detection) reports `Never`, so
239 /// variant-dependent SOVD requests fail fast rather than wait.
240 ///
241 /// # Errors
242 ///
243 /// Returns an error when the transport or a lifecycle hook fails, an
244 /// exclusive disable lease currently owns the state, or the lifecycle
245 /// worker is shutting down.
246 pub async fn enable(&self) -> Result<CommunicationState, CommunicationOperationFailure> {
247 self.run_claimed(LifecycleRequest::Enable).await
248 }
249
250 /// Non-awaiting counterpart to [`enable`](Self::enable), mirroring
251 /// [`request_enable_and_detect`](Self::request_enable_and_detect).
252 #[must_use]
253 pub fn request_enable(&self) -> CommunicationState {
254 self.request_claimed(LifecycleRequest::Enable)
255 }
256
257 /// Runs whole-vehicle variant detection.
258 ///
259 /// The detection stage on its own. It touches neither the transport nor the
260 /// lifecycle hooks and is valid only while `Enabled`. From any other state
261 /// it fails with
262 /// [`TransitionFailure`](CommunicationOperationFailure::TransitionFailure)
263 /// rather than enabling communication first.
264 ///
265 /// Not a lifecycle transition. The runtime stays `Enabled` throughout, so
266 /// single sends keep being served and only multi-request activities are
267 /// held off. A failure also leaves it `Enabled` and publishes
268 /// [`variant_detection`](Self::variant_detection) as `Never`.
269 ///
270 /// # Errors
271 ///
272 /// Returns an error when detection fails, communication is not `Enabled`,
273 /// an exclusive disable lease currently owns the state, a multi-request
274 /// activity guard is currently held (see
275 /// [`CommunicationOperationFailure::GuardsHeld`]), or the lifecycle worker
276 /// is shutting down.
277 pub async fn redetect(&self) -> Result<CommunicationState, CommunicationOperationFailure> {
278 self.run_claimed(LifecycleRequest::Detect).await
279 }
280
281 /// Returns the effective detection policy for the runtime that is currently
282 /// enabled, or being enabled.
283 ///
284 /// `Never` means nothing is currently going to settle any ECU's
285 /// `VariantState`. Consumers use it to answer immediately instead of waiting
286 /// out a readiness timeout. It is not permanent, because an explicit
287 /// [`redetect`](Self::redetect) runs the detector regardless of
288 /// configuration.
289 #[must_use]
290 pub fn variant_detection(&self) -> VariantDetectionMode {
291 self.state.lock().variant_detection
292 }
293
294 async fn run_claimed(
295 &self,
296 request: LifecycleRequest,
297 ) -> Result<CommunicationState, CommunicationOperationFailure> {
298 let operation = request.operation();
299 let mut rx = match self.claim_or_join(request) {
300 ClaimOutcome::Settled(result) => return result,
301 ClaimOutcome::LeaseHeld(_) => {
302 return Err(CommunicationOperationFailure::DisableLeaseHeld { operation });
303 }
304 ClaimOutcome::Pending(rx) => rx,
305 };
306
307 loop {
308 if let Some(result) = rx.borrow_and_update().clone() {
309 return result;
310 }
311 if rx.changed().await.is_err() {
312 return Err(self.fail_closed_enabling_attempt(operation, &rx));
313 }
314 }
315 }
316
317 fn request_claimed(&self, request: LifecycleRequest) -> CommunicationState {
318 match self.claim_or_join(request) {
319 ClaimOutcome::Settled(Ok(settled)) => settled,
320 ClaimOutcome::Settled(Err(failure)) => CommunicationState::Error(failure),
321 ClaimOutcome::LeaseHeld(current) => current,
322 ClaimOutcome::Pending(_) => self.state(),
323 }
324 }
325
326 /// Applies the verdict [`transition::decide`] returns for `request`. All
327 /// joined callers observe the same final result via the returned receiver.
328 fn claim_or_join(&self, request: LifecycleRequest) -> ClaimOutcome {
329 let operation = request.operation();
330 let mut state = self.state.lock();
331 match transition::decide(&state, self.worker.is_shutting_down(), request) {
332 // Detection leaves transport and hooks untouched and writes its own
333 // slot rather than `state.state`.
334 LifecycleDecision::ClaimDetection => self.spawn_redetect(&mut state),
335 LifecycleDecision::JoinDetection => Self::join_detection(&mut state),
336 LifecycleDecision::Claim(shape) => self.spawn_enabling(&mut state, shape),
337 LifecycleDecision::Join(in_flight) => {
338 Self::join_in_flight(&mut state, in_flight, operation)
339 }
340 LifecycleDecision::AlreadyEnabled => {
341 ClaimOutcome::Settled(Ok(CommunicationState::Enabled))
342 }
343 LifecycleDecision::LeaseHeld(current) => ClaimOutcome::LeaseHeld(current),
344 LifecycleDecision::GuardsHeld => {
345 ClaimOutcome::Settled(Err(CommunicationOperationFailure::GuardsHeld { operation }))
346 }
347 LifecycleDecision::ShuttingDown => {
348 ClaimOutcome::Settled(Err(CommunicationOperationFailure::ShuttingDown {
349 operation,
350 }))
351 }
352 // `NotEnabled` covers a `Detect` against a down transport. The
353 // other three are lease verdicts that `decide` never returns here.
354 LifecycleDecision::NotEnabled
355 | LifecycleDecision::ClaimDisable { .. }
356 | LifecycleDecision::SettleDisabled
357 | LifecycleDecision::Conflict => {
358 ClaimOutcome::Settled(Err(CommunicationOperationFailure::TransitionFailure {
359 operation,
360 }))
361 }
362 }
363 }
364
365 fn join_in_flight(
366 state: &mut std::sync::MutexGuard<'_, super::state::CommunicationStateData>,
367 in_flight: CommunicationOperation,
368 requested: CommunicationOperation,
369 ) -> ClaimOutcome {
370 match state.enabling_result.as_ref() {
371 Some(rx) if rx.has_changed().is_ok() => ClaimOutcome::Pending(rx.clone()),
372 Some(_) => {
373 let failure = CommunicationOperationFailure::WorkerUnavailable {
374 operation: in_flight,
375 };
376 state.state = CommunicationState::Error(failure.clone());
377 state.enabling_result = None;
378 ClaimOutcome::Settled(Err(failure))
379 }
380 // Only shutdown clears the result slot while `Enabling`.
381 None => ClaimOutcome::Settled(Err(CommunicationOperationFailure::ShuttingDown {
382 operation: requested,
383 })),
384 }
385 }
386
387 fn spawn_enabling(
388 &self,
389 state: &mut std::sync::MutexGuard<'_, super::state::CommunicationStateData>,
390 operation: CommunicationOperation,
391 ) -> ClaimOutcome {
392 let (tx, rx) = watch::channel(None);
393 let detector = state
394 .runs_detection_for(operation)
395 .then(|| state.variant_detector())
396 .flatten();
397
398 state.state = CommunicationState::Enabling(operation);
399 state.enabling_result = Some(rx.clone());
400 state.variant_detection = detection_mode(detector.as_ref());
401
402 let handle = self.clone();
403 let attempt = rx.clone();
404 cda_interfaces::spawn_named!(
405 &format!("communication-enabling-{operation:?}"),
406 async move {
407 let result = handle
408 .run_enabling_operation(operation, detector, &attempt)
409 .await;
410 let _ = tx.send(Some(result));
411 }
412 );
413 ClaimOutcome::Pending(rx)
414 }
415
416 /// Claims the detection slot and leaves `state.state` alone. The runtime
417 /// stays `Enabled` throughout, so communication that does not depend on the
418 /// variant remains available.
419 fn spawn_redetect(
420 &self,
421 state: &mut std::sync::MutexGuard<'_, super::state::CommunicationStateData>,
422 ) -> ClaimOutcome {
423 let (tx, rx) = watch::channel(None);
424 let detector = state.variant_detector();
425
426 state.detection_in_flight = Some(rx.clone());
427 state.variant_detection = detection_mode(detector.as_ref());
428
429 let handle = self.clone();
430 cda_interfaces::spawn_named!("communication-redetect", async move {
431 let result = handle.run_redetect_operation(detector).await;
432 let result = handle.finish_detection(result);
433 let _ = tx.send(Some(result));
434 });
435 ClaimOutcome::Pending(rx)
436 }
437
438 /// Joins the detection already in flight.
439 ///
440 /// Separate from `join_in_flight`, which reads a different slot. This one
441 /// never publishes a lifecycle state, because a dead detection task leaves
442 /// the runtime `Enabled` just as a failed one does.
443 fn join_detection(
444 state: &mut std::sync::MutexGuard<'_, super::state::CommunicationStateData>,
445 ) -> ClaimOutcome {
446 if let Some(rx) = state
447 .detection_in_flight
448 .as_ref()
449 .filter(|rx| rx.has_changed().is_ok())
450 .cloned()
451 {
452 return ClaimOutcome::Pending(rx);
453 }
454
455 // The sender died without publishing. Clear the slot so the next caller
456 // claims a fresh detection instead of joining a dead one.
457 state.detection_in_flight = None;
458 state.variant_detection = VariantDetectionMode::Never;
459 ClaimOutcome::Settled(Err(CommunicationOperationFailure::WorkerUnavailable {
460 operation: CommunicationOperation::Detect,
461 }))
462 }
463
464 /// Runs the physical operation on the worker, isolating the caller from a
465 /// send/reply failure (worker shutdown mid-flight).
466 ///
467 /// The worker publishes the final state itself before replying (and, on a
468 /// successful activation, calls every hook's `on_enabled()`), so its reply
469 /// is returned unchanged. When the worker cannot be reached (a send
470 /// failure, or a reply channel dropped without an answer), the attempt is
471 /// failed via [`fail_closed_enabling_attempt`](Self::fail_closed_enabling_attempt),
472 /// which only publishes `Error` while `attempt` still owns the state.
473 async fn run_enabling_operation(
474 &self,
475 operation: CommunicationOperation,
476 detector: Option<Arc<dyn CommunicationVariantDetection>>,
477 attempt: &EnablingResultReceiver,
478 ) -> Result<CommunicationState, CommunicationOperationFailure> {
479 let Ok(rx) = self
480 .submit(operation, |reply| LifecycleCommand::Activate {
481 operation,
482 detector,
483 reply,
484 })
485 .await
486 else {
487 return Err(self.fail_closed_enabling_attempt(operation, attempt));
488 };
489 rx.await
490 .unwrap_or_else(|_| Err(self.fail_closed_enabling_attempt(operation, attempt)))
491 }
492
493 /// Runs variant detection on the worker (detector only, no transport
494 /// change), isolating the caller from a send/reply failure.
495 async fn run_redetect_operation(
496 &self,
497 detector: Option<Arc<dyn CommunicationVariantDetection>>,
498 ) -> Result<CommunicationState, CommunicationOperationFailure> {
499 self.submit_and_await(CommunicationOperation::Detect, |reply| {
500 LifecycleCommand::Redetect { detector, reply }
501 })
502 .await
503 }
504
505 /// Clears the detection slot and returns the result unchanged, without
506 /// changing the lifecycle state. On failure it also resets the effective
507 /// detection mode.
508 fn finish_detection(
509 &self,
510 result: Result<CommunicationState, CommunicationOperationFailure>,
511 ) -> Result<CommunicationState, CommunicationOperationFailure> {
512 let mut state = self.state.lock();
513 // Shutdown publishes the terminal state. A detection finishing
514 // alongside it must not overwrite that.
515 if state.shutting_down {
516 return Err(CommunicationOperationFailure::ShuttingDown {
517 operation: CommunicationOperation::Detect,
518 });
519 }
520 state.detection_in_flight = None;
521 if result.is_err() {
522 state.variant_detection = VariantDetectionMode::Never;
523 }
524 result
525 }
526
527 /// Fails the current enabling attempt when the worker could not be reached,
528 /// or the watch channel closed before a result was published.
529 ///
530 /// State is changed only when the current slot belongs to the same channel,
531 /// so neither a stale finalizer nor this fallback can overwrite a newer
532 /// attempt, a result the worker already published, or shutdown's terminal
533 /// state (shutdown clears the slot).
534 fn fail_closed_enabling_attempt(
535 &self,
536 operation: CommunicationOperation,
537 failed: &EnablingResultReceiver,
538 ) -> CommunicationOperationFailure {
539 let failure = self.construct_worker_failure(operation);
540 let mut state = self.state.lock();
541 // Detection changes nothing physical and publishes no lifecycle state,
542 // not even a failed one.
543 if operation == CommunicationOperation::Detect {
544 if state
545 .detection_in_flight
546 .as_ref()
547 .is_some_and(|current| current.same_channel(failed))
548 {
549 state.detection_in_flight = None;
550 state.variant_detection = VariantDetectionMode::Never;
551 }
552 return failure;
553 }
554 if state
555 .enabling_result
556 .as_ref()
557 .is_some_and(|current| current.same_channel(failed))
558 {
559 state.state = CommunicationState::Error(failure.clone());
560 state.enabling_result = None;
561 }
562 failure
563 }
564
565 /// Acquires the sole exclusive disable lease, taking the physical transport
566 /// offline if it was up.
567 ///
568 /// Granted from `Enabled` and from `Disabled`. The lease is exclusive
569 /// ownership of the runtime, not an operation on the transport, so a
570 /// consumer such as a runtime update does not have to bring the vehicle
571 /// network up to become eligible.
572 ///
573 /// `DisableOwner::resumes_transport` records what `release()` means. A lease
574 /// taken from `Enabled` resumes, one taken from `Disabled` returns to
575 /// `Disabled`.
576 ///
577 /// `Enabling(_)`, `Disabling`, `DisabledExclusive` and `Error(_)` are
578 /// conflicts. `Error(_)` has an unknown transport state, so neither resume
579 /// shape can be chosen.
580 ///
581 /// # Errors
582 ///
583 /// Returns an error when communication is in use or cannot be disabled.
584 #[tracing::instrument(skip_all, fields(dlt_context = dlt_ctx!("COMM")))]
585 pub async fn disable(&self, reason: DisableReason) -> Result<DisableLease, DisableError> {
586 let id = DisableLeaseId::new();
587 {
588 let mut state = self.state.lock();
589 match transition::decide(
590 &state,
591 self.worker.is_shutting_down(),
592 LifecycleRequest::Disable,
593 ) {
594 LifecycleDecision::ClaimDisable { resumes_transport } => {
595 // Decide under one lock whether releasing brings the
596 // transport back, and whether it detects when it does.
597 let resume_detects = state.variant_detection == VariantDetectionMode::Always;
598 state.state = CommunicationState::Disabling;
599 state.disable_owner = Some(DisableOwner {
600 id,
601 resumes_transport,
602 resume_detects,
603 });
604 }
605 LifecycleDecision::GuardsHeld => return Err(DisableError::InUse),
606 LifecycleDecision::ShuttingDown => {
607 return Err(DisableError::Failed(
608 CommunicationOperationFailure::ShuttingDown {
609 operation: CommunicationOperation::Disable,
610 },
611 ));
612 }
613 // Every remaining verdict means this state cannot grant a
614 // lease.
615 LifecycleDecision::Conflict
616 | LifecycleDecision::Claim(_)
617 | LifecycleDecision::ClaimDetection
618 | LifecycleDecision::Join(_)
619 | LifecycleDecision::JoinDetection
620 | LifecycleDecision::AlreadyEnabled
621 | LifecycleDecision::SettleDisabled
622 | LifecycleDecision::LeaseHeld(_)
623 | LifecycleDecision::NotEnabled => {
624 tracing::debug!(
625 state = ?state.state,
626 ?reason,
627 "Disable rejected: current state does not permit an exclusive lease"
628 );
629 return Err(DisableError::Conflict);
630 }
631 }
632 }
633
634 match self.spawn_disable(id, reason).await {
635 Ok(Ok(lease)) => Ok(lease),
636 Ok(Err(failure)) => Err(DisableError::Failed(failure)),
637 Err(join_error) => {
638 tracing::error!(%join_error, "Communication disable task failed");
639 let failure = CommunicationOperationFailure::TransitionFailure {
640 operation: CommunicationOperation::Disable,
641 };
642 self.finish_failed_disable_locally(id, failure.clone());
643 Err(DisableError::Failed(failure))
644 }
645 }
646 }
647
648 fn spawn_disable(
649 &self,
650 id: DisableLeaseId,
651 reason: DisableReason,
652 ) -> JoinHandle<Result<DisableLease, CommunicationOperationFailure>> {
653 let handle = self.clone();
654 cda_interfaces::spawn_named!("communication-disable", async move {
655 handle.run_disable_operation(id, reason).await
656 })
657 }
658
659 /// Runs detached from the `disable()` caller so that cancelling the caller
660 /// cannot lose the resulting `DisableLease`. The lease is constructed here,
661 /// so an abandoned task still finishes and the lease's `Drop` still defers
662 /// back to `Disabled`.
663 async fn run_disable_operation(
664 &self,
665 id: DisableLeaseId,
666 reason: DisableReason,
667 ) -> Result<DisableLease, CommunicationOperationFailure> {
668 let operation = CommunicationOperation::Disable;
669 // `submit` rather than `submit_and_await`, because
670 // `finish_failed_disable_locally` must run only for a locally
671 // synthesized failure. The worker's `execute_disable` publishes its own.
672 let rx = match self
673 .submit(operation, |reply| LifecycleCommand::Disable {
674 id,
675 reason,
676 reply,
677 })
678 .await
679 {
680 Ok(rx) => rx,
681 Err(failure) => {
682 self.finish_failed_disable_locally(id, failure.clone());
683 return Err(failure);
684 }
685 };
686 match rx.await {
687 Ok(Ok(())) => Ok(DisableLease::new(self.clone(), id)),
688 Ok(Err(failure)) => Err(failure),
689 Err(_) => {
690 let failure = self.construct_worker_failure(operation);
691 self.finish_failed_disable_locally(id, failure.clone());
692 Err(failure)
693 }
694 }
695 }
696
697 /// Publishes `Error` for a claimed disable whose outcome is unknown because
698 /// the worker could not be reached. A transport failure the worker did
699 /// report is published by `finish_failed_disable` instead.
700 fn finish_failed_disable_locally(
701 &self,
702 id: DisableLeaseId,
703 failure: CommunicationOperationFailure,
704 ) {
705 let mut state = self.state.lock();
706 if state.shutting_down {
707 return;
708 }
709 if DisableOwner::owns(state.disable_owner, id) {
710 state.disable_owner = None;
711 state.state = CommunicationState::Error(failure);
712 }
713 }
714
715 pub(super) async fn submit_release(
716 &self,
717 id: DisableLeaseId,
718 ) -> Result<
719 oneshot::Receiver<Result<CommunicationState, CommunicationOperationFailure>>,
720 CommunicationOperationFailure,
721 > {
722 self.submit(CommunicationOperation::Resume, |reply| {
723 LifecycleCommand::ReleaseDisableLease { id, reply }
724 })
725 .await
726 }
727
728 pub(super) async fn submit_finish(
729 &self,
730 id: DisableLeaseId,
731 ) -> Result<
732 oneshot::Receiver<Result<(), CommunicationOperationFailure>>,
733 CommunicationOperationFailure,
734 > {
735 self.submit(CommunicationOperation::FinishDisableLease, |reply| {
736 LifecycleCommand::FinishDisableLease { id, reply }
737 })
738 .await
739 }
740
741 pub(super) fn defer_disable(
742 &self,
743 id: DisableLeaseId,
744 ) -> Result<(), CommunicationOperationFailure> {
745 let mut state = self.state.lock();
746
747 // Shutdown already clears ownership and publishes the terminal state. A
748 // lease dropped alongside it must not overwrite that.
749 if state.shutting_down {
750 return Ok(());
751 }
752
753 // Not disabled -> no-op
754 let Some(owner) = state.disable_owner else {
755 return Ok(());
756 };
757
758 if owner.id != id {
759 tracing::debug!("Resume permission denied");
760 return Err(CommunicationOperationFailure::PermissionFailure {
761 operation: CommunicationOperation::Resume,
762 });
763 }
764
765 state.disable_owner = None;
766 state.state = CommunicationState::Disabled;
767 Ok(())
768 }
769
770 /// Requests the out-of-band lifecycle shutdown barrier and awaits its
771 /// completion (see `worker`): closes admission, fails queued
772 /// commands deterministically, then best-effort disables the transport.
773 #[tracing::instrument(skip_all, fields(dlt_context = dlt_ctx!("COMM")))]
774 pub async fn shutdown(&self) {
775 // Set under the state lock before the worker starts tearing down. Every
776 // racing state write then either lands before it and is overwritten by
777 // the worker's terminal write, or observes the flag and skips.
778 self.state.lock().shutting_down = true;
779 self.worker.shutdown().await;
780 if let Some(task) = self.worker_task.lock().await.take()
781 && let Err(error) = task.await
782 {
783 tracing::error!(%error, "Communication lifecycle worker terminated unexpectedly");
784 let mut state = self.state.lock();
785 // Only `Enabling` names its own operation. From every other state
786 // the teardown that just failed was the disable-shaped one.
787 let operation = match state.state {
788 CommunicationState::Enabling(operation) => operation,
789 CommunicationState::Enabled
790 | CommunicationState::Disabled
791 | CommunicationState::Disabling
792 | CommunicationState::DisabledExclusive
793 | CommunicationState::RecoveryRequired(_)
794 | CommunicationState::Error(_) => CommunicationOperation::Disable,
795 };
796 state.disable_owner = None;
797 state.enabling_result = None;
798 state.detection_in_flight = None;
799 state.state =
800 CommunicationState::Error(CommunicationOperationFailure::WorkerUnavailable {
801 operation,
802 });
803 }
804 }
805}
806
807/// The constructed authoritative plugin.
808pub struct CommunicationRuntime {
809 /// The selected, `Arc`-wrapped authoritative communication plugin.
810 pub plugin: Arc<dyn CommunicationPlugin>,
811}
812
813struct WorkerBuildGuard(Option<WorkerSender>);
814
815impl Drop for WorkerBuildGuard {
816 fn drop(&mut self) {
817 if let Some(worker) = self.0.take() {
818 worker.request_shutdown();
819 }
820 }
821}
822
823/// Constructs the framework and the startup-selected communication plugin.
824///
825/// The lifecycle worker is started before the plugin builder receives its
826/// fully functional [`CommunicationHandle`].
827///
828/// `init_mode` is passed through unchanged for the plugin to interpret.
829/// `variant_detection` is framework policy, applied by the worker.
830///
831/// # Errors
832///
833/// Returns an error when the plugin builder fails.
834pub async fn build_communication_runtime<B>(
835 builder: B,
836 transport_control: Arc<dyn TransportControl>,
837 init_mode: CommunicationInitMode,
838 variant_detection: VariantDetectionMode,
839) -> Result<CommunicationRuntime, BuildCommunicationRuntimeError<B::Error>>
840where
841 B: CommunicationPluginBuilder,
842{
843 let resources = worker::new_resources(transport_control);
844 let state = Arc::new(CommunicationStateStore::new(
845 CommunicationState::Disabled,
846 variant_detection,
847 ));
848 let (sender, receivers) = worker::channel();
849 let worker_task = Arc::new(tokio::sync::Mutex::new(None));
850 let handle = CommunicationHandle {
851 state: Arc::clone(&state),
852 worker: sender,
853 worker_task: Arc::clone(&worker_task),
854 };
855
856 *worker_task.lock().await = Some(worker::spawn(state, resources, receivers));
857
858 let mut build_guard = WorkerBuildGuard(Some(handle.worker.clone()));
859
860 let built = match builder.build(handle.clone(), init_mode).await {
861 Ok(plugin) => plugin,
862 Err(error) => {
863 handle.shutdown().await;
864 build_guard.0 = None;
865 return Err(BuildCommunicationRuntimeError::Plugin(error));
866 }
867 };
868 build_guard.0 = None;
869
870 let plugin: Arc<dyn CommunicationPlugin> = Arc::new(built);
871
872 Ok(CommunicationRuntime { plugin })
873}
874
875#[cfg(test)]
876mod tests {
877 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
878
879 use async_trait::async_trait;
880 use cda_interfaces::communication_control::{
881 TransportControl, TransportState, error::CommControlError,
882 };
883 use tokio::sync::Notify;
884
885 use super::*;
886 use crate::lifecycle::controller::test_utils::{
887 communication_handle_new, communication_handle_with_detection_mode,
888 };
889
890 /// Test helper: polls state until it matches, with a timeout.
891 async fn poll_until_state(handle: &CommunicationHandle, expected: CommunicationState) {
892 tokio::time::timeout(std::time::Duration::from_secs(5), async {
893 loop {
894 if handle.state() == expected {
895 return;
896 }
897 tokio::task::yield_now().await;
898 }
899 })
900 .await
901 .expect("timed out waiting for expected state");
902 }
903
904 struct RecordingControl {
905 enable_count: AtomicUsize,
906 disable_count: AtomicUsize,
907 fail_enable: AtomicBool,
908 }
909 #[async_trait]
910 impl TransportControl for RecordingControl {
911 async fn enable(&self) -> Result<(), CommControlError> {
912 self.enable_count.fetch_add(1, Ordering::Relaxed);
913 if self.fail_enable.load(Ordering::Relaxed) {
914 Err(CommControlError::InitFailed("failed".to_owned()))
915 } else {
916 Ok(())
917 }
918 }
919 async fn disable(&self) -> Result<(), CommControlError> {
920 self.disable_count.fetch_add(1, Ordering::Relaxed);
921 Ok(())
922 }
923 async fn state(&self) -> TransportState {
924 TransportState::Disabled
925 }
926 }
927 fn handle() -> (CommunicationHandle, Arc<RecordingControl>) {
928 let control = Arc::new(RecordingControl {
929 enable_count: AtomicUsize::new(0),
930 disable_count: AtomicUsize::new(0),
931 fail_enable: AtomicBool::new(false),
932 });
933 (
934 communication_handle_new(Arc::clone(&control) as Arc<dyn TransportControl>),
935 control,
936 )
937 }
938
939 #[derive(Default)]
940 struct BlockingEnableControl {
941 entered: Notify,
942 proceed: Notify,
943 completed: Notify,
944 enable_count: AtomicUsize,
945 disable_count: AtomicUsize,
946 }
947
948 #[async_trait]
949 impl TransportControl for BlockingEnableControl {
950 async fn enable(&self) -> Result<(), CommControlError> {
951 self.enable_count.fetch_add(1, Ordering::Relaxed);
952 self.entered.notify_one();
953 self.proceed.notified().await;
954 self.completed.notify_one();
955 Ok(())
956 }
957
958 async fn disable(&self) -> Result<(), CommControlError> {
959 self.disable_count.fetch_add(1, Ordering::Relaxed);
960 Ok(())
961 }
962
963 async fn state(&self) -> TransportState {
964 TransportState::Disabled
965 }
966 }
967
968 /// A detached fallback write must not overwrite the worker's
969 /// shutdown-sequence write, which would leave `state == Enabled` on a
970 /// torn-down transport.
971 #[tokio::test]
972 async fn shutdown_racing_in_flight_activation_finishes_disabled_not_enabled() {
973 let control = Arc::new(BlockingEnableControl::default());
974 let handle = communication_handle_new(Arc::clone(&control) as Arc<dyn TransportControl>);
975
976 let activation = tokio::spawn({
977 let handle = handle.clone();
978 async move { handle.enable_and_detect().await }
979 });
980 // The worker is now blocked inside transport.enable(), mid-`handle_command`.
981 control.entered.notified().await;
982
983 // Sets `shutting_down` synchronously, then queues behind the worker.
984 let shutdown = tokio::spawn({
985 let handle = handle.clone();
986 async move { handle.shutdown().await }
987 });
988 // Let the shutdown task set the flag before unblocking the transport.
989 tokio::task::yield_now().await;
990 tokio::task::yield_now().await;
991
992 // The worker finishes Activate and the detached task races the
993 // shutdown-sequence write.
994 control.proceed.notify_one();
995
996 assert!(matches!(
997 activation.await.unwrap(),
998 Err(CommunicationOperationFailure::ShuttingDown {
999 operation: CommunicationOperation::EnableAndDetect
1000 })
1001 ));
1002 shutdown.await.unwrap();
1003
1004 assert_eq!(
1005 handle.state(),
1006 CommunicationState::Disabled,
1007 "shutdown must have the last word, never leaving state Enabled after the transport \
1008 was torn down"
1009 );
1010 assert_eq!(control.disable_count.load(Ordering::Relaxed), 1);
1011 assert!(handle.acquire().is_err());
1012 }
1013
1014 #[derive(Default)]
1015 struct BlockingDisableControl {
1016 entered: Notify,
1017 proceed: Notify,
1018 disable_completed: Notify,
1019 resume_entered: Notify,
1020 resume_proceed: Notify,
1021 enable_count: AtomicUsize,
1022 disable_count: AtomicUsize,
1023 }
1024
1025 #[async_trait]
1026 impl TransportControl for BlockingDisableControl {
1027 async fn enable(&self) -> Result<(), CommControlError> {
1028 if self.enable_count.fetch_add(1, Ordering::Relaxed) == 1 {
1029 self.resume_entered.notify_one();
1030 self.resume_proceed.notified().await;
1031 }
1032 Ok(())
1033 }
1034
1035 async fn disable(&self) -> Result<(), CommControlError> {
1036 self.disable_count.fetch_add(1, Ordering::Relaxed);
1037 self.entered.notify_one();
1038 self.proceed.notified().await;
1039 self.disable_completed.notify_one();
1040 Ok(())
1041 }
1042
1043 async fn state(&self) -> TransportState {
1044 TransportState::Enabled
1045 }
1046 }
1047
1048 #[tokio::test]
1049 async fn activation_joins_in_flight_lease_resume() {
1050 let control = Arc::new(BlockingDisableControl::default());
1051 let handle = communication_handle_new(Arc::clone(&control) as Arc<dyn TransportControl>);
1052 assert_eq!(
1053 handle.enable_and_detect().await,
1054 Ok(CommunicationState::Enabled)
1055 );
1056
1057 let disable = tokio::spawn({
1058 let handle = handle.clone();
1059 async move { handle.disable(DisableReason::RuntimeUpdate).await }
1060 });
1061 control.entered.notified().await;
1062 control.proceed.notify_one();
1063 let lease = disable.await.unwrap().unwrap();
1064
1065 let release = tokio::spawn(async move { lease.release().await });
1066 control.resume_entered.notified().await;
1067 assert_eq!(
1068 handle.state(),
1069 CommunicationState::Enabling(CommunicationOperation::Resume)
1070 );
1071
1072 let activation = tokio::spawn({
1073 let handle = handle.clone();
1074 async move { handle.enable_and_detect().await }
1075 });
1076 tokio::task::yield_now().await;
1077 control.resume_proceed.notify_one();
1078
1079 assert_eq!(release.await.unwrap(), Ok(CommunicationState::Enabled));
1080 assert_eq!(activation.await.unwrap(), Ok(CommunicationState::Enabled));
1081 assert_eq!(control.enable_count.load(Ordering::Relaxed), 2);
1082 }
1083
1084 /// A lease is granted from `Disabled` too, and releasing it returns to
1085 /// `Disabled`. A release that activated would let a consumer holding only
1086 /// `DisableCommunication` start communication `init_mode` never
1087 /// authorized.
1088 #[tokio::test]
1089 async fn lease_from_disabled_touches_no_transport_and_releases_back_to_disabled() {
1090 let (handle, control) = handle();
1091 assert_eq!(handle.state(), CommunicationState::Disabled);
1092
1093 let lease = handle
1094 .disable(DisableReason::RuntimeUpdate)
1095 .await
1096 .expect("a lease must be granted from a deferred runtime");
1097 assert_eq!(handle.state(), CommunicationState::DisabledExclusive);
1098 assert_eq!(
1099 control.disable_count.load(Ordering::Relaxed),
1100 0,
1101 "nothing was up, so nothing may be taken down"
1102 );
1103
1104 // Exclusivity still holds while the lease is out.
1105 assert!(matches!(
1106 handle.disable(DisableReason::RuntimeUpdate).await,
1107 Err(DisableError::Conflict)
1108 ));
1109 assert_eq!(
1110 handle.enable_and_detect().await,
1111 Err(CommunicationOperationFailure::DisableLeaseHeld {
1112 operation: CommunicationOperation::EnableAndDetect
1113 })
1114 );
1115
1116 assert_eq!(lease.release().await, Ok(CommunicationState::Disabled));
1117 assert_eq!(handle.state(), CommunicationState::Disabled);
1118 assert_eq!(
1119 control.enable_count.load(Ordering::Relaxed),
1120 0,
1121 "releasing must never enable a runtime the releaser did not find enabled"
1122 );
1123 }
1124
1125 /// A lease over a runtime that was never enabled must not deinitialize its
1126 /// hooks. `deinitialize()` without a preceding `initialize()` breaks
1127 /// `CommunicationLifecycle`'s paired contract.
1128 #[tokio::test]
1129 async fn lease_from_disabled_does_not_deinitialize_uninitialized_hooks() {
1130 let (handle, _control) = handle();
1131 let hook = Arc::new(DeinitTrackingInitializer {
1132 fail: AtomicBool::new(false),
1133 init_calls: AtomicUsize::new(0),
1134 deinit_calls: AtomicUsize::new(0),
1135 });
1136 handle
1137 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
1138 .await
1139 .expect("registration must succeed");
1140
1141 let lease = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
1142 assert_eq!(hook.init_calls.load(Ordering::Relaxed), 0);
1143 assert_eq!(
1144 hook.deinit_calls.load(Ordering::Relaxed),
1145 0,
1146 "a hook that never initialized must not be deinitialized"
1147 );
1148
1149 assert_eq!(lease.release().await, Ok(CommunicationState::Disabled));
1150 assert_eq!(hook.init_calls.load(Ordering::Relaxed), 0);
1151 assert_eq!(hook.deinit_calls.load(Ordering::Relaxed), 0);
1152 }
1153
1154 /// Dropping a lease taken from `Disabled` is the same no-op resume as
1155 /// releasing it.
1156 #[tokio::test]
1157 async fn dropping_a_lease_from_disabled_leaves_it_disabled() {
1158 let (handle, control) = handle();
1159 let lease = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
1160 drop(lease);
1161 assert_eq!(handle.state(), CommunicationState::Disabled);
1162 assert_eq!(control.enable_count.load(Ordering::Relaxed), 0);
1163 assert_eq!(control.disable_count.load(Ordering::Relaxed), 0);
1164 }
1165
1166 /// `Error(_)` has an unknown transport state, so no lease is granted from
1167 /// it.
1168 #[tokio::test]
1169 async fn lease_refused_from_error() {
1170 let (handle, control) = handle();
1171 control.fail_enable.store(true, Ordering::Relaxed);
1172 assert!(handle.enable_and_detect().await.is_err());
1173 assert!(matches!(handle.state(), CommunicationState::Error(_)));
1174
1175 assert!(matches!(
1176 handle.disable(DisableReason::RuntimeUpdate).await,
1177 Err(DisableError::Conflict)
1178 ));
1179 }
1180
1181 #[tokio::test]
1182 async fn stale_lease_cannot_resume_a_later_disable() {
1183 let (handle, _) = handle();
1184 assert_eq!(
1185 handle.enable_and_detect().await,
1186 Ok(CommunicationState::Enabled)
1187 );
1188 let first = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
1189 let stale_id = first.identity().unwrap();
1190 assert_eq!(first.release().await, Ok(CommunicationState::Enabled));
1191
1192 let second = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
1193 // The stale id must be rejected without disturbing the current lease.
1194 let reply = handle.submit_release(stale_id).await.unwrap();
1195 assert_eq!(
1196 reply.await.unwrap(),
1197 Err(CommunicationOperationFailure::TransitionFailure {
1198 operation: CommunicationOperation::Resume,
1199 })
1200 );
1201 assert_eq!(handle.state(), CommunicationState::DisabledExclusive);
1202 assert_eq!(second.release().await, Ok(CommunicationState::Enabled));
1203 }
1204
1205 #[tokio::test]
1206 async fn stale_lease_finish_reports_the_exact_finish_operation() {
1207 let (handle, _) = handle();
1208 let first = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
1209 let stale_id = first.identity().unwrap();
1210 assert_eq!(first.finish().await, Ok(()));
1211
1212 let second = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
1213 let reply = handle.submit_finish(stale_id).await.unwrap();
1214 assert_eq!(
1215 reply.await.unwrap(),
1216 Err(CommunicationOperationFailure::TransitionFailure {
1217 operation: CommunicationOperation::FinishDisableLease,
1218 })
1219 );
1220 assert_eq!(handle.state(), CommunicationState::DisabledExclusive);
1221 assert_eq!(second.finish().await, Ok(()));
1222 }
1223
1224 /// A `release()` future abandoned before its first poll must still defer. It
1225 /// owns the lease and never reached the `self.id = None` that hands
1226 /// ownership to the worker, so dropping it runs the lease's `Drop`.
1227 #[tokio::test]
1228 async fn dropping_unpolled_release_future_defers_synchronously() {
1229 let (handle, control) = handle();
1230 assert_eq!(
1231 handle.enable_and_detect().await,
1232 Ok(CommunicationState::Enabled)
1233 );
1234 let lease = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
1235
1236 let release_future = lease.release();
1237 drop(release_future);
1238
1239 // Asserted without polling. Deferring must not need a later await.
1240 assert_eq!(handle.state(), CommunicationState::Disabled);
1241 assert_eq!(control.enable_count.load(Ordering::Relaxed), 1);
1242 }
1243
1244 /// A release that reaches the worker after teardown has begun must refuse
1245 /// rather than bring the transport back up.
1246 ///
1247 /// The flag is set directly rather than via `shutdown()`, which would also
1248 /// close admission and refuse the release at the mailbox.
1249 #[tokio::test]
1250 async fn release_reaching_the_worker_after_shutdown_began_does_not_resume() {
1251 let (handle, control) = handle();
1252 assert_eq!(
1253 handle.enable_and_detect().await,
1254 Ok(CommunicationState::Enabled)
1255 );
1256 let lease = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
1257 assert_eq!(handle.state(), CommunicationState::DisabledExclusive);
1258 let enables_before = control.enable_count.load(Ordering::Relaxed);
1259
1260 handle.state.lock().shutting_down = true;
1261
1262 assert!(
1263 matches!(
1264 lease.release().await,
1265 Err(CommunicationOperationFailure::ShuttingDown {
1266 operation: CommunicationOperation::Resume
1267 })
1268 ),
1269 "a release racing teardown must report shutdown, not a stale lease"
1270 );
1271 assert_eq!(
1272 control.enable_count.load(Ordering::Relaxed),
1273 enables_before,
1274 "the transport must not be re-enabled once teardown has begun"
1275 );
1276 // The refused release must not publish a state of its own.
1277 assert_eq!(handle.state(), CommunicationState::DisabledExclusive);
1278 }
1279
1280 #[tokio::test]
1281 async fn shutdown_fails_queued_commands_deterministically() {
1282 let (handle, _control) = handle();
1283 assert_eq!(
1284 handle.enable_and_detect().await,
1285 Ok(CommunicationState::Enabled)
1286 );
1287
1288 handle.shutdown().await;
1289
1290 assert!(matches!(
1291 handle.enable_and_detect().await,
1292 Err(CommunicationOperationFailure::ShuttingDown { .. })
1293 ));
1294 assert!(matches!(
1295 handle.disable(DisableReason::RuntimeUpdate).await,
1296 Err(DisableError::Failed(
1297 CommunicationOperationFailure::ShuttingDown { .. }
1298 ))
1299 ));
1300 assert_eq!(
1301 handle
1302 .register_lifecycle_hook(Arc::new(CountingInitializer))
1303 .await,
1304 Err(CommunicationOperationFailure::ShuttingDown {
1305 operation: CommunicationOperation::RegisterLifecycleHook,
1306 })
1307 );
1308 }
1309
1310 struct CountingInitializer;
1311 #[async_trait]
1312 impl CommunicationLifecycle for CountingInitializer {
1313 fn name(&self) -> &'static str {
1314 "counting"
1315 }
1316 async fn initialize(&self) -> Result<(), CommControlError> {
1317 Ok(())
1318 }
1319 async fn on_enabled(&self) {}
1320 }
1321
1322 #[tokio::test]
1323 async fn registration_reports_unavailable_worker_after_unexpected_exit() {
1324 let (handle, _control) = handle();
1325 let task = handle
1326 .worker_task
1327 .lock()
1328 .await
1329 .take()
1330 .expect("test handle must own a worker task");
1331 task.abort();
1332 assert!(task.await.unwrap_err().is_cancelled());
1333
1334 assert_eq!(
1335 handle
1336 .register_lifecycle_hook(Arc::new(CountingInitializer))
1337 .await,
1338 Err(CommunicationOperationFailure::WorkerUnavailable {
1339 operation: CommunicationOperation::RegisterLifecycleHook,
1340 })
1341 );
1342 }
1343
1344 /// Admission backpressures instead of dropping commands. With the mailbox
1345 /// filled to `LIFECYCLE_WORKER_CAPACITY` and nothing draining it, a further
1346 /// registration blocks until capacity frees up, and is then served.
1347 #[tokio::test]
1348 async fn admission_blocks_at_capacity_instead_of_losing_commands() {
1349 let control = Arc::new(BlockingEnableControl::default());
1350 let handle = communication_handle_new(Arc::clone(&control) as Arc<dyn TransportControl>);
1351
1352 // Occupy the worker with a slow Activate so nothing drains the mailbox.
1353 let activation = tokio::spawn({
1354 let handle = handle.clone();
1355 async move { handle.enable_and_detect().await }
1356 });
1357 control.entered.notified().await;
1358
1359 // Fill the mailbox exactly to capacity. Uses the raw sender, because
1360 // `register_lifecycle_hook` also awaits the worker's reply, which never
1361 // arrives while the worker is blocked.
1362 for _ in 0..worker::LIFECYCLE_WORKER_CAPACITY {
1363 let (reply, _rx) = oneshot::channel();
1364 tokio::time::timeout(
1365 std::time::Duration::from_secs(1),
1366 handle.worker.send(LifecycleCommand::RegisterLifecycleHook {
1367 initializer: Arc::new(CountingInitializer),
1368 reply,
1369 }),
1370 )
1371 .await
1372 .expect("filling to capacity must not block")
1373 .unwrap();
1374 }
1375
1376 // One more admission must block. Through the public API, because it is
1377 // the handle's admission path that has to backpressure.
1378 let over_capacity = handle.register_lifecycle_hook(Arc::new(CountingInitializer));
1379 tokio::pin!(over_capacity);
1380 assert!(
1381 tokio::time::timeout(std::time::Duration::from_millis(100), &mut over_capacity)
1382 .await
1383 .is_err(),
1384 "admission beyond capacity must block, not be accepted immediately"
1385 );
1386
1387 // Unblocking the worker drains the mailbox and the pending admission is
1388 // served.
1389 control.proceed.notify_one();
1390 tokio::time::timeout(std::time::Duration::from_secs(5), over_capacity)
1391 .await
1392 .expect("blocked admission must be admitted once capacity frees up")
1393 .expect("the admitted registration must succeed");
1394
1395 activation.await.unwrap().unwrap();
1396 }
1397
1398 /// Concurrent `activate()` callers arriving while an activation is in flight
1399 /// must all join it and observe the same result.
1400 #[tokio::test]
1401 async fn concurrent_activate_calls_join_one_operation_and_share_result() {
1402 let control = Arc::new(BlockingEnableControl::default());
1403 let handle = communication_handle_new(Arc::clone(&control) as Arc<dyn TransportControl>);
1404
1405 let first = tokio::spawn({
1406 let handle = handle.clone();
1407 async move { handle.enable_and_detect().await }
1408 });
1409 control.entered.notified().await;
1410
1411 // These arrive while the first activation is in flight.
1412 let joiners: Vec<_> = (0..4)
1413 .map(|_| {
1414 tokio::spawn({
1415 let handle = handle.clone();
1416 async move { handle.enable_and_detect().await }
1417 })
1418 })
1419 .collect();
1420 assert_eq!(
1421 handle.state(),
1422 CommunicationState::Enabling(CommunicationOperation::EnableAndDetect)
1423 );
1424
1425 control.proceed.notify_one();
1426 assert_eq!(first.await.unwrap(), Ok(CommunicationState::Enabled));
1427 for joiner in joiners {
1428 assert_eq!(joiner.await.unwrap(), Ok(CommunicationState::Enabled));
1429 }
1430
1431 assert_eq!(control.enable_count.load(Ordering::Relaxed), 1);
1432 }
1433
1434 /// `request_activate` called while communication is `Error` must claim and
1435 /// retry rather than return the stale `Error` state.
[docs]1436 /// [[ test~deferred-activation-retry, Failed deferred activation can be retried, test ]]
1437 #[tokio::test]
1438 async fn repeated_failure_returns_to_error_and_is_retried_again() {
1439 let (handle, control) = handle();
1440 control.fail_enable.store(true, Ordering::Relaxed);
1441 assert!(handle.enable_and_detect().await.is_err());
1442 assert!(matches!(handle.state(), CommunicationState::Error(_)));
1443
1444 assert_eq!(
1445 handle.request_enable_and_detect(),
1446 CommunicationState::Enabling(CommunicationOperation::EnableAndDetect)
1447 );
1448 tokio::time::timeout(std::time::Duration::from_secs(5), async {
1449 loop {
1450 if matches!(handle.state(), CommunicationState::Error(_)) {
1451 return;
1452 }
1453 tokio::task::yield_now().await;
1454 }
1455 })
1456 .await
1457 .expect("timed out waiting for Error state");
1458 assert_eq!(control.enable_count.load(Ordering::Relaxed), 2);
1459
1460 control.fail_enable.store(false, Ordering::Relaxed);
1461 assert_eq!(
1462 handle.request_enable_and_detect(),
1463 CommunicationState::Enabling(CommunicationOperation::EnableAndDetect)
1464 );
1465 poll_until_state(&handle, CommunicationState::Enabled).await;
1466 assert_eq!(control.enable_count.load(Ordering::Relaxed), 3);
1467 }
1468
1469 /// `disable` rejects active guards, grants exactly one exclusive lease and
1470 /// blocks re-activation while it is held. Only the matching lease may
1471 /// re-activate. Dropping it leaves communication unowned `Disabled`.
1472 #[tokio::test]
1473 async fn disable_rejects_active_guards_and_blocks_activate_while_leased() {
1474 let (handle, control) = handle();
1475 assert_eq!(
1476 handle.enable_and_detect().await,
1477 Ok(CommunicationState::Enabled)
1478 );
1479
1480 let guard = handle.acquire().unwrap();
1481 assert!(matches!(
1482 handle.disable(DisableReason::RuntimeUpdate).await,
1483 Err(DisableError::InUse)
1484 ));
1485 drop(guard);
1486
1487 let lease = handle
1488 .disable(DisableReason::RuntimeUpdate)
1489 .await
1490 .expect("disable must succeed once no guard is active");
1491 assert_eq!(handle.state(), CommunicationState::DisabledExclusive);
1492 assert_eq!(control.disable_count.load(Ordering::Relaxed), 1);
1493
1494 assert_eq!(
1495 handle.enable_and_detect().await,
1496 Err(CommunicationOperationFailure::DisableLeaseHeld {
1497 operation: CommunicationOperation::EnableAndDetect
1498 })
1499 );
1500 assert_eq!(
1501 handle.request_enable_and_detect(),
1502 CommunicationState::DisabledExclusive
1503 );
1504
1505 assert!(matches!(
1506 handle.disable(DisableReason::RuntimeUpdate).await,
1507 Err(DisableError::Conflict)
1508 ));
1509
1510 assert_eq!(lease.release().await, Ok(CommunicationState::Enabled));
1511 assert_eq!(handle.state(), CommunicationState::Enabled);
1512 assert_eq!(control.enable_count.load(Ordering::Relaxed), 2);
1513 }
1514
1515 /// `finish` must never enable the transport, even though this lease was taken
1516 /// from an enabled runtime.
1517 #[tokio::test]
1518 async fn finish_stays_disabled_and_leaves_the_runtime_reusable() {
1519 let (handle, control) = handle();
1520 assert_eq!(
1521 handle.enable_and_detect().await,
1522 Ok(CommunicationState::Enabled)
1523 );
1524 assert_eq!(control.enable_count.load(Ordering::Relaxed), 1);
1525
1526 let lease = handle
1527 .disable(DisableReason::RuntimeUpdate)
1528 .await
1529 .expect("disable must succeed once no guard is active");
1530 assert_eq!(handle.state(), CommunicationState::DisabledExclusive);
1531
1532 assert_eq!(lease.finish().await, Ok(()));
1533
1534 assert_eq!(handle.state(), CommunicationState::Disabled);
1535 assert_eq!(
1536 control.enable_count.load(Ordering::Relaxed),
1537 1,
1538 "finish must not re-enable the transport"
1539 );
1540
1541 assert_eq!(
1542 handle.enable_and_detect().await,
1543 Ok(CommunicationState::Enabled)
1544 );
1545 assert_eq!(control.enable_count.load(Ordering::Relaxed), 2);
1546 }
1547
1548 /// A lease taken from an already-disabled runtime also stays disabled under
1549 /// `finish`.
1550 #[tokio::test]
1551 async fn finish_from_already_disabled_stays_disabled() {
1552 let (handle, _control) = handle();
1553
1554 let lease = handle
1555 .disable(DisableReason::RuntimeUpdate)
1556 .await
1557 .expect("disable must succeed from Disabled");
1558 assert_eq!(handle.state(), CommunicationState::DisabledExclusive);
1559
1560 assert_eq!(lease.finish().await, Ok(()));
1561
1562 assert_eq!(handle.state(), CommunicationState::Disabled);
1563 }
1564
1565 #[derive(Default)]
1566 struct DeinitTrackingInitializer {
1567 fail: AtomicBool,
1568 init_calls: AtomicUsize,
1569 deinit_calls: AtomicUsize,
1570 }
1571
1572 #[async_trait]
1573 impl CommunicationLifecycle for DeinitTrackingInitializer {
1574 fn name(&self) -> &'static str {
1575 "deinit-tracking"
1576 }
1577 async fn initialize(&self) -> Result<(), CommControlError> {
1578 self.init_calls.fetch_add(1, Ordering::Relaxed);
1579 if self.fail.load(Ordering::Relaxed) {
1580 Err(CommControlError::InitFailed("simulated failure".to_owned()))
1581 } else {
1582 Ok(())
1583 }
1584 }
1585 async fn on_enabled(&self) {}
1586 async fn deinitialize(&self) {
1587 self.deinit_calls.fetch_add(1, Ordering::Relaxed);
1588 }
1589 }
1590
1591 /// Shutdown must notify registered hooks before tearing down the transport,
1592 /// so keep-alive work started in `initialize()` does not outlive it.
1593 #[tokio::test]
1594 async fn shutdown_deinitializes_registered_hooks() {
1595 let (handle, _control) = handle();
1596 let initializer = Arc::new(DeinitTrackingInitializer::default());
1597 handle
1598 .register_lifecycle_hook(Arc::clone(&initializer) as Arc<dyn CommunicationLifecycle>)
1599 .await
1600 .expect("registration must succeed");
1601 assert_eq!(
1602 handle.enable_and_detect().await,
1603 Ok(CommunicationState::Enabled)
1604 );
1605 assert_eq!(initializer.deinit_calls.load(Ordering::Relaxed), 0);
1606
1607 handle.shutdown().await;
1608
1609 assert_eq!(initializer.deinit_calls.load(Ordering::Relaxed), 1);
1610 }
1611
1612 /// When a later initializer fails during activation, the ones that already
1613 /// succeeded must be deinitialized before the transport goes back down.
1614 #[tokio::test]
1615 async fn activation_rollback_deinitializes_succeeded_prefix() {
1616 let (handle, _control) = handle();
1617 let first = Arc::new(DeinitTrackingInitializer::default());
1618 let second = Arc::new(DeinitTrackingInitializer {
1619 fail: AtomicBool::new(true),
1620 ..Default::default()
1621 });
1622 handle
1623 .register_lifecycle_hook(Arc::clone(&first) as Arc<dyn CommunicationLifecycle>)
1624 .await
1625 .expect("registration must succeed");
1626 handle
1627 .register_lifecycle_hook(Arc::clone(&second) as Arc<dyn CommunicationLifecycle>)
1628 .await
1629 .expect("registration must succeed");
1630
1631 assert!(matches!(
1632 handle.enable_and_detect().await,
1633 Err(CommunicationOperationFailure::InitializerFailure { .. })
1634 ));
1635
1636 assert_eq!(
1637 first.deinit_calls.load(Ordering::Relaxed),
1638 1,
1639 "the already-succeeded initializer must be deinitialized on rollback"
1640 );
1641 assert_eq!(
1642 second.deinit_calls.load(Ordering::Relaxed),
1643 0,
1644 "the failing initializer was never successfully initialized, so it is not \
1645 deinitialized"
1646 );
1647 }
1648
1649 // Variant detection
1650
1651 struct FailingDetector {
1652 fail: AtomicBool,
1653 calls: AtomicUsize,
1654 }
1655
1656 impl FailingDetector {
1657 fn new(fail: bool) -> Arc<Self> {
1658 Arc::new(Self {
1659 fail: AtomicBool::new(fail),
1660 calls: AtomicUsize::new(0),
1661 })
1662 }
1663 }
1664
1665 #[async_trait]
1666 impl CommunicationVariantDetection for FailingDetector {
1667 fn name(&self) -> &'static str {
1668 "failing-detector"
1669 }
1670 async fn detect(&self) -> Result<(), CommControlError> {
1671 self.calls.fetch_add(1, Ordering::Relaxed);
1672 if self.fail.load(Ordering::Relaxed) {
1673 Err(CommControlError::InitFailed("simulated failure".to_owned()))
1674 } else {
1675 Ok(())
1676 }
1677 }
1678 }
1679
1680 struct BlockingDetector {
1681 entered: Notify,
1682 proceed: Notify,
1683 }
1684 #[async_trait]
1685 impl CommunicationVariantDetection for BlockingDetector {
1686 fn name(&self) -> &'static str {
1687 "blocking-detector"
1688 }
1689 async fn detect(&self) -> Result<(), CommControlError> {
1690 self.entered.notify_one();
1691 self.proceed.notified().await;
1692 Ok(())
1693 }
1694 }
1695
1696 /// Registers a detector that always succeeds and reports its call count.
1697 async fn with_detector(handle: &CommunicationHandle) -> Arc<FailingDetector> {
1698 let detector = FailingDetector::new(false);
1699 handle
1700 .register_variant_detection(
1701 Arc::clone(&detector) as Arc<dyn CommunicationVariantDetection>
1702 )
1703 .await
1704 .expect("detector registration must succeed");
1705 detector
1706 }
1707
1708 /// `enable()` brings the transport up and initializes the lifecycle hooks, but runs no detection.
1709 #[tokio::test]
1710 async fn enable_runs_lifecycle_hooks_but_not_detection() {
1711 let (handle, control) = handle();
1712 let hook = Arc::new(DeinitTrackingInitializer {
1713 fail: AtomicBool::new(false),
1714 init_calls: AtomicUsize::new(0),
1715 deinit_calls: AtomicUsize::new(0),
1716 });
1717 handle
1718 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
1719 .await
1720 .expect("registration must succeed");
1721 let detector = with_detector(&handle).await;
1722
1723 assert_eq!(handle.enable().await, Ok(CommunicationState::Enabled));
1724
1725 assert_eq!(control.enable_count.load(Ordering::Relaxed), 1);
1726 assert_eq!(
1727 hook.init_calls.load(Ordering::Relaxed),
1728 1,
1729 "a lifecycle hook follows the transport and must run on every enable"
1730 );
1731 assert_eq!(
1732 detector.calls.load(Ordering::Relaxed),
1733 0,
1734 "enable() must not run variant detection"
1735 );
1736 assert_eq!(
1737 handle.variant_detection(),
1738 VariantDetectionMode::Never,
1739 "an enable-only runtime must report that nothing will settle a variant"
1740 );
1741 }
1742
1743 /// `activate()` runs all three stages, and reports that detection is live.
1744 #[tokio::test]
1745 async fn activate_runs_detection_after_hooks() {
1746 let (handle, control) = handle();
1747 let detector = with_detector(&handle).await;
1748
1749 assert_eq!(
1750 handle.enable_and_detect().await,
1751 Ok(CommunicationState::Enabled)
1752 );
1753 assert_eq!(control.enable_count.load(Ordering::Relaxed), 1);
1754 assert_eq!(detector.calls.load(Ordering::Relaxed), 1);
1755 assert_eq!(handle.variant_detection(), VariantDetectionMode::Always);
1756 }
1757
1758 /// `redetect` while `Enabled` runs the detector without calling
1759 /// `TransportControl::enable()` again or re-entering a lifecycle hook's
1760 /// `initialize()`.
1761 #[tokio::test]
1762 async fn redetect_runs_detector_without_reinitializing_hooks() {
1763 let (handle, control) = handle();
1764 let hook = Arc::new(DeinitTrackingInitializer {
1765 fail: AtomicBool::new(false),
1766 init_calls: AtomicUsize::new(0),
1767 deinit_calls: AtomicUsize::new(0),
1768 });
1769 handle
1770 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
1771 .await
1772 .expect("registration must succeed");
1773 let detector = with_detector(&handle).await;
1774
1775 assert_eq!(
1776 handle.enable_and_detect().await,
1777 Ok(CommunicationState::Enabled)
1778 );
1779 assert_eq!(handle.redetect().await, Ok(CommunicationState::Enabled));
1780
1781 assert_eq!(
1782 detector.calls.load(Ordering::Relaxed),
1783 2,
1784 "re-detection must run the detector again"
1785 );
1786 assert_eq!(
1787 hook.init_calls.load(Ordering::Relaxed),
1788 1,
1789 "re-detection must not re-initialize lifecycle hooks: there was no deinitialize \
1790 between, and the trait promises the calls come in pairs"
1791 );
1792 assert_eq!(
1793 control.enable_count.load(Ordering::Relaxed),
1794 1,
1795 "re-detection must never call TransportControl::enable() again"
1796 );
1797 assert_eq!(control.disable_count.load(Ordering::Relaxed), 0);
1798 }
1799
1800 /// Detection is refused while an activity guard is held. That guard is the
1801 /// only lock spanning a whole multi-request sequence.
1802 #[tokio::test]
1803 async fn redetect_refused_while_activity_guard_held() {
1804 let (handle, control) = handle();
1805 assert_eq!(
1806 handle.enable_and_detect().await,
1807 Ok(CommunicationState::Enabled)
1808 );
1809
1810 let guard = handle.acquire().unwrap();
1811 assert_eq!(
1812 handle.redetect().await,
1813 Err(CommunicationOperationFailure::GuardsHeld {
1814 operation: CommunicationOperation::Detect
1815 })
1816 );
1817 assert_eq!(handle.state(), CommunicationState::Enabled);
1818 drop(guard);
1819
1820 // Once released, detection proceeds.
1821 assert_eq!(handle.redetect().await, Ok(CommunicationState::Enabled));
1822 assert_eq!(control.enable_count.load(Ordering::Relaxed), 1);
1823 }
1824
1825 /// `redetect` is the detection stage alone, so from `Disabled` and `Error`
1826 /// it refuses rather than widening itself into an activation.
1827 #[tokio::test]
1828 async fn detect_rejected_when_not_enabled() {
1829 let (handle, control) = handle();
1830 let detector = with_detector(&handle).await;
1831
1832 assert_eq!(
1833 handle.redetect().await,
1834 Err(CommunicationOperationFailure::TransitionFailure {
1835 operation: CommunicationOperation::Detect
1836 })
1837 );
1838 assert_eq!(
1839 control.enable_count.load(Ordering::Relaxed),
1840 0,
1841 "a refused detection must not touch the transport"
1842 );
1843 assert_eq!(detector.calls.load(Ordering::Relaxed), 0);
1844 assert_eq!(handle.state(), CommunicationState::Disabled);
1845
1846 // Same refusal from `Error`.
1847 control.fail_enable.store(true, Ordering::Relaxed);
1848 assert!(handle.enable_and_detect().await.is_err());
1849 assert!(matches!(handle.state(), CommunicationState::Error(_)));
1850 assert_eq!(
1851 handle.redetect().await,
1852 Err(CommunicationOperationFailure::TransitionFailure {
1853 operation: CommunicationOperation::Detect
1854 })
1855 );
1856 }
1857
1858 /// Concurrent detections join the in-flight attempt and observe the same
1859 /// result rather than claiming a second one.
1860 #[tokio::test]
1861 async fn concurrent_redetect_calls_join_one_operation_and_share_result() {
1862 let control = Arc::new(BlockingEnableControl {
1863 entered: Notify::new(),
1864 proceed: Notify::new(),
1865 completed: Notify::new(),
1866 enable_count: AtomicUsize::new(0),
1867 disable_count: AtomicUsize::new(0),
1868 });
1869 let handle = communication_handle_new(Arc::clone(&control) as Arc<dyn TransportControl>);
1870 // Reach Enabled first, so the blocking below comes from the slow
1871 // detector rather than the transport enable.
1872 control.proceed.notify_one();
1873 assert_eq!(
1874 handle.enable_and_detect().await,
1875 Ok(CommunicationState::Enabled)
1876 );
1877
1878 let detector = Arc::new(BlockingDetector {
1879 entered: Notify::new(),
1880 proceed: Notify::new(),
1881 });
1882 handle
1883 .register_variant_detection(
1884 Arc::clone(&detector) as Arc<dyn CommunicationVariantDetection>
1885 )
1886 .await
1887 .expect("registration must succeed");
1888
1889 let first = tokio::spawn({
1890 let handle = handle.clone();
1891 async move { handle.redetect().await }
1892 });
1893 detector.entered.notified().await;
1894
1895 let joiners: Vec<_> = (0..4)
1896 .map(|_| {
1897 tokio::spawn({
1898 let handle = handle.clone();
1899 async move { handle.redetect().await }
1900 })
1901 })
1902 .collect();
1903 // The runtime stays `Enabled` throughout variant detection, so guards
1904 // keep being granted. Moving to `Enabling` would fail this acquisition.
1905 assert_eq!(handle.state(), CommunicationState::Enabled);
1906 let guard = handle
1907 .acquire()
1908 .expect("diagnostics must keep being served during a sweep");
1909 drop(guard);
1910
1911 detector.proceed.notify_one();
1912 assert_eq!(first.await.unwrap(), Ok(CommunicationState::Enabled));
1913 for joiner in joiners {
1914 assert_eq!(joiner.await.unwrap(), Ok(CommunicationState::Enabled));
1915 }
1916 assert_eq!(control.enable_count.load(Ordering::Relaxed), 1);
1917 }
1918
1919 /// A detector failure during a `Detect` leaves the transport enabled. The
1920 /// operation did not bring it up, so it does not tear it down.
1921 #[tokio::test]
1922 async fn redetect_detection_failure_leaves_runtime_enabled_and_transport_up() {
1923 let (handle, control) = handle();
1924 assert_eq!(
1925 handle.enable_and_detect().await,
1926 Ok(CommunicationState::Enabled)
1927 );
1928
1929 let detector = FailingDetector::new(true);
1930 handle
1931 .register_variant_detection(
1932 Arc::clone(&detector) as Arc<dyn CommunicationVariantDetection>
1933 )
1934 .await
1935 .expect("registration must succeed");
1936
1937 assert!(matches!(
1938 handle.redetect().await,
1939 Err(CommunicationOperationFailure::DetectionFailure { .. })
1940 ));
1941 assert_eq!(
1942 handle.state(),
1943 CommunicationState::Enabled,
1944 "a failed re-detection changed nothing physical, so the runtime is still enabled - \
1945 publishing `Error` here would claim a transport that is up is down"
1946 );
1947 assert_eq!(
1948 handle.variant_detection(),
1949 VariantDetectionMode::Never,
1950 "nothing is going to settle a variant now, and the readiness gate has to be able to \
1951 say so instead of advertising a sweep that is never coming"
1952 );
1953 assert_eq!(
1954 control.disable_count.load(Ordering::Relaxed),
1955 0,
1956 "a failed re-detection must not disable the transport it did not enable itself"
1957 );
1958 assert_eq!(control.enable_count.load(Ordering::Relaxed), 1);
1959
1960 assert!(handle.acquire().is_ok());
1961 assert!(handle.acquire().is_ok());
1962 }
1963
1964 /// A failed detection must not publish `Error(_)`. `Error` means the
1965 /// transport is down, so `decide` would admit a recovery activation and
1966 /// `run_activation` would re-run `initialize()` on hooks that are still
1967 /// initialized. Staying `Enabled` makes the readiness gate's
1968 /// `request_activate` settle as `AlreadyEnabled` instead.
1969 #[tokio::test]
1970 async fn failed_redetect_does_not_let_recovery_reinitialize_live_hooks() {
1971 let (handle, control) = handle();
1972 let hook = Arc::new(DeinitTrackingInitializer {
1973 fail: AtomicBool::new(false),
1974 init_calls: AtomicUsize::new(0),
1975 deinit_calls: AtomicUsize::new(0),
1976 });
1977 handle
1978 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
1979 .await
1980 .expect("registration must succeed");
1981 assert_eq!(
1982 handle.enable_and_detect().await,
1983 Ok(CommunicationState::Enabled)
1984 );
1985 assert_eq!(hook.init_calls.load(Ordering::Relaxed), 1);
1986
1987 let detector = FailingDetector::new(true);
1988 handle
1989 .register_variant_detection(
1990 Arc::clone(&detector) as Arc<dyn CommunicationVariantDetection>
1991 )
1992 .await
1993 .expect("registration must succeed");
1994 assert!(handle.redetect().await.is_err());
1995
1996 // What the readiness gate does on the next ECU request.
1997 assert_eq!(
1998 handle.request_enable_and_detect(),
1999 CommunicationState::Enabled
2000 );
2001 assert_eq!(
2002 handle.enable_and_detect().await,
2003 Ok(CommunicationState::Enabled)
2004 );
2005
2006 assert_eq!(
2007 hook.init_calls.load(Ordering::Relaxed),
2008 1,
2009 "initialize() must not run a second time without a deinitialize() between - the trait \
2010 documents implementations as not having to be re-entrant"
2011 );
2012 assert_eq!(hook.deinit_calls.load(Ordering::Relaxed), 0);
2013 assert_eq!(
2014 control.enable_count.load(Ordering::Relaxed),
2015 1,
2016 "nothing was claimed, so no stage ran at all"
2017 );
2018 }
2019
2020 /// A lease must not be granted while variant detection runs. The state stays
2021 /// `Enabled` throughout, so `decide` checks the detection slot explicitly.
2022 #[tokio::test]
2023 async fn disable_is_refused_while_a_redetection_is_in_flight() {
2024 // Only the detector needs to block. The lease resume at the end has to
2025 // be able to enable again without deadlocking.
2026 let (handle, control) = handle();
2027 assert_eq!(
2028 handle.enable_and_detect().await,
2029 Ok(CommunicationState::Enabled)
2030 );
2031
2032 let detector = Arc::new(BlockingDetector {
2033 entered: Notify::new(),
2034 proceed: Notify::new(),
2035 });
2036 handle
2037 .register_variant_detection(
2038 Arc::clone(&detector) as Arc<dyn CommunicationVariantDetection>
2039 )
2040 .await
2041 .expect("registration must succeed");
2042
2043 let sweep = tokio::spawn({
2044 let handle = handle.clone();
2045 async move { handle.redetect().await }
2046 });
2047 detector.entered.notified().await;
2048
2049 assert!(
2050 matches!(
2051 handle.disable(DisableReason::RuntimeUpdate).await,
2052 Err(DisableError::InUse)
2053 ),
2054 "detection is a use of communication, so a lease must wait for it"
2055 );
2056 assert_eq!(
2057 control.disable_count.load(Ordering::Relaxed),
2058 0,
2059 "the transport must not be taken down underneath a running sweep"
2060 );
2061
2062 detector.proceed.notify_one();
2063 assert_eq!(sweep.await.unwrap(), Ok(CommunicationState::Enabled));
2064
2065 // Dropped rather than released. A resume would repeat the detecting
2066 // shape and re-enter the blocking detector with nobody left to unblock
2067 // it.
2068 let lease = handle
2069 .disable(DisableReason::RuntimeUpdate)
2070 .await
2071 .expect("the lease must be grantable once detection settles");
2072 drop(lease);
2073 assert_eq!(handle.state(), CommunicationState::Disabled);
2074 }
2075
2076 /// A detector failure during a full activation unwinds all hooks and takes
2077 /// the transport back down, since this operation brought it up.
2078 #[tokio::test]
2079 async fn detector_failure_deinitializes_hooks_and_downs_transport() {
2080 let (handle, control) = handle();
2081 let hook = Arc::new(DeinitTrackingInitializer {
2082 fail: AtomicBool::new(false),
2083 init_calls: AtomicUsize::new(0),
2084 deinit_calls: AtomicUsize::new(0),
2085 });
2086 handle
2087 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
2088 .await
2089 .expect("registration must succeed");
2090 let detector = FailingDetector::new(true);
2091 handle
2092 .register_variant_detection(
2093 Arc::clone(&detector) as Arc<dyn CommunicationVariantDetection>
2094 )
2095 .await
2096 .expect("registration must succeed");
2097
2098 assert!(matches!(
2099 handle.enable_and_detect().await,
2100 Err(CommunicationOperationFailure::DetectionFailure { .. })
2101 ));
2102 assert_eq!(hook.init_calls.load(Ordering::Relaxed), 1);
2103 assert_eq!(
2104 hook.deinit_calls.load(Ordering::Relaxed),
2105 1,
2106 "every successfully initialized hook must be unwound when detection fails"
2107 );
2108 assert_eq!(control.disable_count.load(Ordering::Relaxed), 1);
2109 }
2110
2111 /// Releasing a disable lease repeats the shape of the enable it resumes, so
2112 /// an enable-only runtime does not become a detecting one.
2113 #[tokio::test]
2114 async fn enable_only_runtime_resumes_without_detection() {
2115 let (handle, control) = handle();
2116 let hook = Arc::new(DeinitTrackingInitializer {
2117 fail: AtomicBool::new(false),
2118 init_calls: AtomicUsize::new(0),
2119 deinit_calls: AtomicUsize::new(0),
2120 });
2121 handle
2122 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
2123 .await
2124 .expect("registration must succeed");
2125 let detector = with_detector(&handle).await;
2126
2127 assert_eq!(handle.enable().await, Ok(CommunicationState::Enabled));
2128 let lease = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
2129 assert_eq!(lease.release().await, Ok(CommunicationState::Enabled));
2130
2131 assert_eq!(
2132 detector.calls.load(Ordering::Relaxed),
2133 0,
2134 "resuming an enable-only runtime must not start detecting behind the plugin's back"
2135 );
2136 assert_eq!(
2137 hook.init_calls.load(Ordering::Relaxed),
2138 2,
2139 "hooks follow the transport, so a resume re-initializes them"
2140 );
2141 assert_eq!(control.enable_count.load(Ordering::Relaxed), 2);
2142 assert_eq!(handle.variant_detection(), VariantDetectionMode::Never);
2143 }
2144
2145 /// A lease restores the runtime it displaced, not the one that was first
2146 /// brought up.
2147 ///
2148 /// The resume shape is captured from the effective detection mode when the
2149 /// lease is granted. A lease taken after an explicit `redetect()` therefore
2150 /// resumes detecting rather than repeating the original `enable()`.
2151 #[tokio::test]
2152 async fn lease_granted_after_redetect_resumes_detecting() {
2153 let (handle, _control) = handle();
2154 let detector = with_detector(&handle).await;
2155
2156 assert_eq!(handle.enable().await, Ok(CommunicationState::Enabled));
2157 assert_eq!(handle.variant_detection(), VariantDetectionMode::Never);
2158
2159 assert_eq!(handle.redetect().await, Ok(CommunicationState::Enabled));
2160 assert_eq!(detector.calls.load(Ordering::Relaxed), 1);
2161 assert_eq!(handle.variant_detection(), VariantDetectionMode::Always);
2162
2163 let lease = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
2164 assert_eq!(lease.release().await, Ok(CommunicationState::Enabled));
2165
2166 assert_eq!(
2167 detector.calls.load(Ordering::Relaxed),
2168 2,
2169 "the lease displaced a detecting runtime, so releasing it detects again"
2170 );
2171 assert_eq!(handle.variant_detection(), VariantDetectionMode::Always);
2172 }
2173
2174 /// Every stage runs only when the operation claims a transition. From
2175 /// `Enabled` there is nothing to claim, so `enable_and_detect()` runs no
2176 /// detector. Detection against a live transport is `redetect()`.
2177 #[tokio::test]
2178 async fn enable_and_detect_does_not_detect_an_already_enabled_runtime() {
2179 let (handle, _control) = handle();
2180 let detector = with_detector(&handle).await;
2181
2182 assert_eq!(handle.enable().await, Ok(CommunicationState::Enabled));
2183 assert_eq!(detector.calls.load(Ordering::Relaxed), 0);
2184 assert_eq!(handle.variant_detection(), VariantDetectionMode::Never);
2185
2186 assert_eq!(
2187 handle.enable_and_detect().await,
2188 Ok(CommunicationState::Enabled)
2189 );
2190 assert_eq!(
2191 detector.calls.load(Ordering::Relaxed),
2192 0,
2193 "an already-enabled runtime claims nothing, so nothing detects"
2194 );
2195 assert_eq!(
2196 handle.variant_detection(),
2197 VariantDetectionMode::Never,
2198 "and the reported policy still says nothing will settle a variant"
2199 );
2200
2201 assert_eq!(handle.redetect().await, Ok(CommunicationState::Enabled));
2202 assert_eq!(detector.calls.load(Ordering::Relaxed), 1);
2203 assert_eq!(handle.variant_detection(), VariantDetectionMode::Always);
2204 }
2205
2206 /// `VariantDetectionMode::Never` gates the detector stage of an activation
2207 /// only, never the transport, the lifecycle hooks or an explicit
2208 /// `redetect()`.
2209 #[tokio::test]
2210 async fn variant_detection_never_config_skips_detector_not_hooks() {
2211 let control = Arc::new(RecordingControl {
2212 enable_count: AtomicUsize::new(0),
2213 disable_count: AtomicUsize::new(0),
2214 fail_enable: AtomicBool::new(false),
2215 });
2216 let handle = communication_handle_with_detection_mode(
2217 Arc::clone(&control) as Arc<dyn TransportControl>,
2218 VariantDetectionMode::Never,
2219 );
2220 let hook = Arc::new(DeinitTrackingInitializer {
2221 fail: AtomicBool::new(false),
2222 init_calls: AtomicUsize::new(0),
2223 deinit_calls: AtomicUsize::new(0),
2224 });
2225 handle
2226 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
2227 .await
2228 .expect("registration must succeed");
2229 let detector = with_detector(&handle).await;
2230
2231 assert_eq!(
2232 handle.enable_and_detect().await,
2233 Ok(CommunicationState::Enabled)
2234 );
2235 assert_eq!(control.enable_count.load(Ordering::Relaxed), 1);
2236 assert_eq!(
2237 hook.init_calls.load(Ordering::Relaxed),
2238 1,
2239 "the configured detection policy must not gate lifecycle hooks"
2240 );
2241 assert_eq!(
2242 detector.calls.load(Ordering::Relaxed),
2243 0,
2244 "activation must not detect automatically in Never mode"
2245 );
2246 assert_eq!(handle.variant_detection(), VariantDetectionMode::Never);
2247
2248 assert_eq!(handle.redetect().await, Ok(CommunicationState::Enabled));
2249 assert_eq!(
2250 detector.calls.load(Ordering::Relaxed),
2251 1,
2252 "an explicit detection must run even where automatic detection is configured off"
2253 );
2254 assert_eq!(handle.variant_detection(), VariantDetectionMode::Always);
2255 }
2256
2257 /// With no detector registered nothing settles a variant, whatever the
2258 /// configuration says, and the runtime reports `Never`.
2259 #[tokio::test]
2260 async fn variant_detection_reports_never_without_a_registered_detector() {
2261 let (handle, _control) = handle();
2262 assert_eq!(
2263 handle.enable_and_detect().await,
2264 Ok(CommunicationState::Enabled)
2265 );
2266 assert_eq!(handle.variant_detection(), VariantDetectionMode::Never);
2267 }
2268
2269 /// Lifecycle events recorded by test hooks.
2270 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
2271 enum LifecycleEvent {
2272 Initialized,
2273 Enabled,
2274 Deinitialized,
2275 }
2276
2277 /// Records lifecycle call order into a shared, cloneable log.
2278 #[derive(Clone, Default)]
2279 struct RecordingHook {
2280 events: Arc<std::sync::Mutex<Vec<LifecycleEvent>>>,
2281 }
2282
2283 #[async_trait]
2284 impl CommunicationLifecycle for RecordingHook {
2285 fn name(&self) -> &'static str {
2286 "recording"
2287 }
2288 async fn initialize(&self) -> Result<(), CommControlError> {
2289 self.events
2290 .lock()
2291 .unwrap()
2292 .push(LifecycleEvent::Initialized);
2293 Ok(())
2294 }
2295 async fn on_enabled(&self) {
2296 self.events.lock().unwrap().push(LifecycleEvent::Enabled);
2297 }
2298 async fn deinitialize(&self) {
2299 self.events
2300 .lock()
2301 .unwrap()
2302 .push(LifecycleEvent::Deinitialized);
2303 }
2304 }
2305
2306 /// `on_enabled` must observe the state already published as `Enabled`, and
2307 /// must be able to acquire a guard: both are false while still `Enabling`.
2308 #[tokio::test]
2309 async fn on_enabled_runs_after_enabled_is_published() {
2310 struct ObservingHook {
2311 handle: CommunicationHandle,
2312 observed_state: Arc<std::sync::Mutex<Option<CommunicationState>>>,
2313 observed_acquire_ok: Arc<AtomicBool>,
2314 }
2315 #[async_trait]
2316 impl CommunicationLifecycle for ObservingHook {
2317 fn name(&self) -> &'static str {
2318 "observing"
2319 }
2320 async fn initialize(&self) -> Result<(), CommControlError> {
2321 Ok(())
2322 }
2323 async fn on_enabled(&self) {
2324 *self.observed_state.lock().unwrap() = Some(self.handle.state());
2325 self.observed_acquire_ok
2326 .store(self.handle.acquire().is_ok(), Ordering::Relaxed);
2327 }
2328 }
2329
2330 let (handle, _control) = handle();
2331 let observed_state = Arc::new(std::sync::Mutex::new(None));
2332 let observed_acquire_ok = Arc::new(AtomicBool::new(false));
2333 handle
2334 .register_lifecycle_hook(Arc::new(ObservingHook {
2335 handle: handle.clone(),
2336 observed_state: Arc::clone(&observed_state),
2337 observed_acquire_ok: Arc::clone(&observed_acquire_ok),
2338 }) as Arc<dyn CommunicationLifecycle>)
2339 .await
2340 .expect("registration must succeed");
2341
2342 assert_eq!(
2343 handle.enable_and_detect().await,
2344 Ok(CommunicationState::Enabled)
2345 );
2346
2347 assert_eq!(
2348 *observed_state.lock().unwrap(),
2349 Some(CommunicationState::Enabled),
2350 "on_enabled must observe the state already published as Enabled"
2351 );
2352 assert!(
2353 observed_acquire_ok.load(Ordering::Relaxed),
2354 "on_enabled must be able to acquire a guard once Enabled is published"
2355 );
2356 }
2357
2358 /// By the time an activation call returns, every registered hook's
2359 /// `on_enabled` has already run.
2360 #[tokio::test]
2361 async fn on_enabled_runs_before_activation_caller_returns() {
2362 let (handle, _control) = handle();
2363 let hook = Arc::new(RecordingHook::default());
2364 handle
2365 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
2366 .await
2367 .expect("registration must succeed");
2368
2369 assert_eq!(
2370 handle.enable_and_detect().await,
2371 Ok(CommunicationState::Enabled)
2372 );
2373
2374 assert_eq!(
2375 *hook.events.lock().unwrap(),
2376 vec![LifecycleEvent::Initialized, LifecycleEvent::Enabled]
2377 );
2378 }
2379
2380 /// A hook that blocks inside `on_enabled` until released.
2381 #[derive(Default)]
2382 struct BlockingOnEnabledHook {
2383 events: Arc<std::sync::Mutex<Vec<LifecycleEvent>>>,
2384 entered_on_enabled: Notify,
2385 proceed_on_enabled: Notify,
2386 }
2387
2388 #[async_trait]
2389 impl CommunicationLifecycle for BlockingOnEnabledHook {
2390 fn name(&self) -> &'static str {
2391 "blocking-on-enabled"
2392 }
2393 async fn initialize(&self) -> Result<(), CommControlError> {
2394 self.events
2395 .lock()
2396 .unwrap()
2397 .push(LifecycleEvent::Initialized);
2398 Ok(())
2399 }
2400 async fn on_enabled(&self) {
2401 self.entered_on_enabled.notify_one();
2402 self.proceed_on_enabled.notified().await;
2403 self.events.lock().unwrap().push(LifecycleEvent::Enabled);
2404 }
2405 async fn deinitialize(&self) {
2406 self.events
2407 .lock()
2408 .unwrap()
2409 .push(LifecycleEvent::Deinitialized);
2410 }
2411 }
2412
2413 /// A disable requested while `on_enabled` is still running must not
2414 /// deinitialize hooks until `on_enabled` finishes. `Enabled` is already
2415 /// published by the time `on_enabled` starts, so a disable can be claimed
2416 /// concurrently; the worker handles one lifecycle command at a time, so the
2417 /// claimed `Disable` cannot reach `execute_disable` before `on_enabled`
2418 /// returns.
2419 #[tokio::test]
2420 async fn disable_requested_during_on_enabled_deinitializes_afterwards() {
2421 let (handle, _control) = handle();
2422 let hook = Arc::new(BlockingOnEnabledHook::default());
2423 handle
2424 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
2425 .await
2426 .expect("registration must succeed");
2427
2428 let activation = tokio::spawn({
2429 let handle = handle.clone();
2430 async move { handle.enable_and_detect().await }
2431 });
2432 hook.entered_on_enabled.notified().await;
2433
2434 let disable = tokio::spawn({
2435 let handle = handle.clone();
2436 async move { handle.disable(DisableReason::RuntimeUpdate).await }
2437 });
2438 tokio::time::timeout(std::time::Duration::from_secs(1), async {
2439 while handle.state() != CommunicationState::Disabling {
2440 tokio::task::yield_now().await;
2441 }
2442 })
2443 .await
2444 .expect("disable must claim Disabling while on_enabled is still running");
2445
2446 hook.proceed_on_enabled.notify_one();
2447
2448 assert_eq!(activation.await.unwrap(), Ok(CommunicationState::Enabled));
2449 let lease = disable
2450 .await
2451 .unwrap()
2452 .expect("disable must succeed once on_enabled finishes");
2453
2454 assert_eq!(
2455 *hook.events.lock().unwrap(),
2456 vec![
2457 LifecycleEvent::Initialized,
2458 LifecycleEvent::Enabled,
2459 LifecycleEvent::Deinitialized,
2460 ],
2461 "deinitialize must never run before on_enabled finishes"
2462 );
2463 drop(lease);
2464 }
2465
2466 /// `on_enabled` must not run when a later initializer fails: the
2467 /// activation never reached `Enabled`.
2468 #[tokio::test]
2469 async fn on_enabled_not_called_when_initializer_fails() {
2470 let (handle, _control) = handle();
2471 let failing = Arc::new(DeinitTrackingInitializer {
2472 fail: AtomicBool::new(true),
2473 ..Default::default()
2474 });
2475 let hook = Arc::new(RecordingHook::default());
2476 handle
2477 .register_lifecycle_hook(Arc::clone(&failing) as Arc<dyn CommunicationLifecycle>)
2478 .await
2479 .expect("registration must succeed");
2480 handle
2481 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
2482 .await
2483 .expect("registration must succeed");
2484
2485 assert!(matches!(
2486 handle.enable_and_detect().await,
2487 Err(CommunicationOperationFailure::InitializerFailure { .. })
2488 ));
2489
2490 assert!(
2491 hook.events.lock().unwrap().is_empty(),
2492 "on_enabled must not run when an earlier initializer failed"
2493 );
2494 }
2495
2496 /// `on_enabled` must not run when detection fails: the activation never
2497 /// reached `Enabled`.
2498 #[tokio::test]
2499 async fn on_enabled_not_called_when_detection_fails() {
2500 let (handle, _control) = handle();
2501 let hook = Arc::new(RecordingHook::default());
2502 handle
2503 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
2504 .await
2505 .expect("registration must succeed");
2506 let detector = FailingDetector::new(true);
2507 handle
2508 .register_variant_detection(
2509 Arc::clone(&detector) as Arc<dyn CommunicationVariantDetection>
2510 )
2511 .await
2512 .expect("registration must succeed");
2513
2514 assert!(matches!(
2515 handle.enable_and_detect().await,
2516 Err(CommunicationOperationFailure::DetectionFailure { .. })
2517 ));
2518
2519 assert_eq!(
2520 *hook.events.lock().unwrap(),
2521 vec![LifecycleEvent::Initialized, LifecycleEvent::Deinitialized],
2522 "a failed detection unwinds the hook that already initialized, but must not call \
2523 on_enabled on it"
2524 );
2525 }
2526
2527 /// A redetect never deinitializes hooks, so it must not call `on_enabled`
2528 /// either: the state was `Enabled` throughout and the hook was never
2529 /// deinitialized.
2530 #[tokio::test]
2531 async fn on_enabled_not_called_on_redetect() {
2532 let (handle, _control) = handle();
2533 let hook = Arc::new(RecordingHook::default());
2534 handle
2535 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
2536 .await
2537 .expect("registration must succeed");
2538 let detector = with_detector(&handle).await;
2539
2540 assert_eq!(
2541 handle.enable_and_detect().await,
2542 Ok(CommunicationState::Enabled)
2543 );
2544 assert_eq!(
2545 *hook.events.lock().unwrap(),
2546 vec![LifecycleEvent::Initialized, LifecycleEvent::Enabled]
2547 );
2548 assert_eq!(detector.calls.load(Ordering::Relaxed), 1);
2549
2550 assert_eq!(handle.redetect().await, Ok(CommunicationState::Enabled));
2551 assert_eq!(detector.calls.load(Ordering::Relaxed), 2);
2552
2553 assert_eq!(
2554 *hook.events.lock().unwrap(),
2555 vec![LifecycleEvent::Initialized, LifecycleEvent::Enabled],
2556 "redetect must not call on_enabled again"
2557 );
2558 }
2559
2560 /// Releasing an exclusive disable lease that resumes communication calls
2561 /// `on_enabled` too, just like a fresh activation.
2562 #[tokio::test]
2563 async fn on_enabled_called_on_lease_release_resume() {
2564 let (handle, _control) = handle();
2565 let hook = Arc::new(RecordingHook::default());
2566 handle
2567 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
2568 .await
2569 .expect("registration must succeed");
2570
2571 assert_eq!(
2572 handle.enable_and_detect().await,
2573 Ok(CommunicationState::Enabled)
2574 );
2575 assert_eq!(
2576 *hook.events.lock().unwrap(),
2577 vec![LifecycleEvent::Initialized, LifecycleEvent::Enabled]
2578 );
2579
2580 let lease = handle.disable(DisableReason::RuntimeUpdate).await.unwrap();
2581 assert_eq!(
2582 *hook.events.lock().unwrap(),
2583 vec![
2584 LifecycleEvent::Initialized,
2585 LifecycleEvent::Enabled,
2586 LifecycleEvent::Deinitialized,
2587 ]
2588 );
2589
2590 assert_eq!(lease.release().await, Ok(CommunicationState::Enabled));
2591
2592 assert_eq!(
2593 *hook.events.lock().unwrap(),
2594 vec![
2595 LifecycleEvent::Initialized,
2596 LifecycleEvent::Enabled,
2597 LifecycleEvent::Deinitialized,
2598 LifecycleEvent::Initialized,
2599 LifecycleEvent::Enabled,
2600 ],
2601 "releasing a lease that resumes communication must call on_enabled again"
2602 );
2603 }
2604
2605 /// Shutdown racing an in-flight activation must win the publish, and
2606 /// `on_enabled` must then not run. Companion to
2607 /// `shutdown_racing_in_flight_activation_finishes_disabled_not_enabled`,
2608 /// which asserts the resulting state.
2609 #[tokio::test]
2610 async fn on_enabled_not_called_when_shutdown_wins_publish() {
2611 let control = Arc::new(BlockingEnableControl::default());
2612 let handle = communication_handle_new(Arc::clone(&control) as Arc<dyn TransportControl>);
2613 let hook = Arc::new(RecordingHook::default());
2614 handle
2615 .register_lifecycle_hook(Arc::clone(&hook) as Arc<dyn CommunicationLifecycle>)
2616 .await
2617 .expect("registration must succeed");
2618
2619 let activation = tokio::spawn({
2620 let handle = handle.clone();
2621 async move { handle.enable_and_detect().await }
2622 });
2623 control.entered.notified().await;
2624
2625 let shutdown = tokio::spawn({
2626 let handle = handle.clone();
2627 async move { handle.shutdown().await }
2628 });
2629 tokio::task::yield_now().await;
2630 tokio::task::yield_now().await;
2631
2632 control.proceed.notify_one();
2633
2634 assert!(matches!(
2635 activation.await.unwrap(),
2636 Err(CommunicationOperationFailure::ShuttingDown { .. })
2637 ));
2638 shutdown.await.unwrap();
2639
2640 assert!(
2641 !hook
2642 .events
2643 .lock()
2644 .unwrap()
2645 .contains(&LifecycleEvent::Enabled),
2646 "on_enabled must not run when shutdown won the publish for this activation"
2647 );
2648 }
2649
2650 /// When the worker cannot be reached at all, the controller still
2651 /// publishes the failure itself, through `fail_closed_enabling_attempt`, for
2652 /// a send failure or a dropped reply.
2653 #[tokio::test]
2654 async fn worker_unreachable_still_publishes_error() {
2655 let (handle, _control) = handle();
2656 let task = handle
2657 .worker_task
2658 .lock()
2659 .await
2660 .take()
2661 .expect("test handle must own a worker task");
2662 task.abort();
2663 assert!(task.await.unwrap_err().is_cancelled());
2664
2665 assert_eq!(
2666 handle.enable_and_detect().await,
2667 Err(CommunicationOperationFailure::WorkerUnavailable {
2668 operation: CommunicationOperation::EnableAndDetect,
2669 })
2670 );
2671 assert!(matches!(handle.state(), CommunicationState::Error(_)));
2672 }
2673
2674 /// The worker publishes `Enabled` before it calls `on_enabled()`. If it
2675 /// then dies before replying (here: a hook panics), the caller sees the
2676 /// dropped reply, but the fallback must not overwrite the state the worker
2677 /// already published.
2678 #[tokio::test]
2679 async fn fallback_does_not_overwrite_a_result_the_worker_published() {
2680 struct PanicOnEnabled;
2681
2682 #[async_trait]
2683 impl CommunicationLifecycle for PanicOnEnabled {
2684 fn name(&self) -> &'static str {
2685 "panic-on-enabled"
2686 }
2687 async fn initialize(&self) -> Result<(), CommControlError> {
2688 Ok(())
2689 }
2690 async fn on_enabled(&self) {
2691 panic!("hook panicked in on_enabled");
2692 }
2693 }
2694
2695 let (handle, _control) = handle();
2696 handle
2697 .register_lifecycle_hook(Arc::new(PanicOnEnabled) as Arc<dyn CommunicationLifecycle>)
2698 .await
2699 .expect("registration must succeed");
2700
2701 assert_eq!(
2702 handle.enable_and_detect().await,
2703 Err(CommunicationOperationFailure::WorkerUnavailable {
2704 operation: CommunicationOperation::EnableAndDetect,
2705 })
2706 );
2707 assert_eq!(
2708 handle.state(),
2709 CommunicationState::Enabled,
2710 "the fallback must not overwrite the result the worker published"
2711 );
2712 }
2713}