From 9ea0384145c5372c88da329c8cfd341bac92ad39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignas=20Vy=C5=A1niauskas?= Date: Mon, 25 Nov 2024 14:03:44 +0200 Subject: [PATCH 1/3] Split Update state into version info, system status and process state This refactors Update state into 3 parts: - `process_state`, which indicates what action the Update process is currently performing (checking, downloading, installing or sleeping) - `system_status`, which reflects what the Update process thinks is the current system state (up-to-date, needs-update, etc) - `version_info`, which reflects the current slot and latest upstream versions. The reason for doing this is to decompose the process transitions from the (determined) system state, as consumed by other components via the `React.Signal`. For example, this allows to observe the last `system_status` in the GUI while the Update process is performing other tasks. Note that only the `process_state` is an input state (meaning that the behaviour of Update process depends on it), whereas the `system_status` and `version_info` are "output" states. In theory, the basic FSM step could be refactored into something like: val run_step: process_state -> (process_state * (system_status option) * (version_info option)) Lwt.t where the recursive loop would now `run_rec` maintain the `state` and update it's fields if the output `option`s are set. However, this splits the logic over two functions rather than one and makes it more complicated to interpret state machine. --- controller/server/gui.ml | 4 +- controller/server/update.ml | 178 ++++++++++-------- controller/server/update.mli | 38 +++- controller/server/view/status_page.ml | 15 +- controller/tests/server/update/outcome.ml | 31 +-- controller/tests/server/update/scenario.ml | 20 +- .../tests/server/update/update_prop_tests.ml | 6 +- .../tests/server/update/update_tests.ml | 170 ++++++++++------- 8 files changed, 269 insertions(+), 193 deletions(-) diff --git a/controller/server/gui.ml b/controller/server/gui.ml index 5968fb22..fa908262 100644 --- a/controller/server/gui.ml +++ b/controller/server/gui.ml @@ -424,12 +424,12 @@ module StatusGui = struct ) in reboot () - let get_status ~health_s ~update_s ~rauc = + let get_status ~health_s ~(update_s:Update.state React.signal) ~rauc = let health_state = health_s |> Lwt_react.S.value in let update_state = update_s |> Lwt_react.S.value in let%lwt booted_slot = Rauc.get_booted_slot rauc in let%lwt rauc = - match update_state with + match update_state.process_state with (* RAUC status is not meaningful while installing https://github.com/rauc/rauc/issues/416 *) diff --git a/controller/server/update.ml b/controller/server/update.ml index 544cbb13..005fb599 100644 --- a/controller/server/update.ml +++ b/controller/server/update.ml @@ -25,25 +25,38 @@ let sexp_of_version_info v = List [Atom "inactive"; Atom (Semver.to_string v.inactive)]; ]) - - -type state = - | GettingVersionInfo +type update_error = | ErrorGettingVersionInfo of string - | UpToDate of version_info - | Downloading of string | ErrorDownloading of string - | Installing of string | ErrorInstalling of string - (* inactive system has been updated and reboot is required to boot into updated system *) +[@@deriving sexp_of] + +type system_status = + | UpToDate + | NeedsUpdate | RebootRequired - (* inactive system is up to date, but current system was selected for boot *) | OutOfDateVersionSelected - (* there are no known-good systems and a reinstall is recommended *) | ReinstallRequired + | UpdateError of update_error [@@deriving sexp_of] type sleep_duration = float (* seconds *) +[@@deriving sexp_of] + +(** State of update mechanism *) +type process_state = + | Sleeping of sleep_duration + | GettingVersionInfo + | Downloading of string + | Installing of string +[@@deriving sexp_of] + +type state = { + version_info: version_info option; + system_status: system_status; + process_state: process_state +} +[@@deriving sexp_of] type config = { error_backoff_duration: sleep_duration; @@ -57,7 +70,7 @@ module type ServiceDeps = sig end module type UpdateService = sig - val run : set_state:(state -> unit) -> state -> unit Lwt.t + val run : (state -> unit) -> unit Lwt.t val run_step : state -> state Lwt.t end @@ -72,7 +85,7 @@ let evaluate_version_info current_primary booted_slot version_info = (* Should not happen during the automatic update process (one partition must always be older than latest upstream), but can happen if e.g. a newer version is manually installed into one of the slots. *) - UpToDate version_info + UpToDate else if booted_up_to_date || inactive_up_to_date then match current_primary with | Some primary_slot -> @@ -80,7 +93,7 @@ let evaluate_version_info current_primary booted_slot version_info = (* Don't care if inactive can be updated. I.e. Only update the inactive partition once the booted partition is outdated. This results in always two versions being available on the machine. *) - UpToDate version_info + UpToDate else if booted_slot = primary_slot then (* Inactive is up to date while booted is out of date, but booted was @@ -95,8 +108,7 @@ let evaluate_version_info current_primary booted_slot version_info = ReinstallRequired else - (* Both systems are out of date -> update the inactive system *) - Downloading (Semver.to_string version_info.latest) + NeedsUpdate (** Helper to parse semver from string or fail *) @@ -110,15 +122,15 @@ let semver_of_string string = | Some version -> version +let initial_state = { + version_info = None; + system_status = UpToDate; (* start with assuming a good state *) + process_state = GettingVersionInfo; +} + module Make(Deps : ServiceDeps) : UpdateService = struct open Deps - let sleep_error_backoff () = - Lwt_unix.sleep config.error_backoff_duration - - let sleep_update_check () = - Lwt_unix.sleep config.check_for_updates_interval - (** Get version information *) let get_version_info () = let%lwt latest = ClientI.get_latest_version () >|= semver_of_string in @@ -142,10 +154,13 @@ module Make(Deps : ServiceDeps) : UpdateService = struct |> return (* Update mechanism process *) - (** perform a single state transition from given state *) + (** Perform a single state transition from given state + + Note: the action performed depends _only_ on the input + state.process_state. + *) let run_step (state:state) : state Lwt.t = - let set = Lwt.return in - match (state) with + match (state.process_state) with | GettingVersionInfo -> (* get version information and decide what to do *) let%lwt resp = Lwt_result.catch (fun () -> @@ -156,82 +171,84 @@ module Make(Deps : ServiceDeps) : UpdateService = struct ) in (match resp with | Ok (slot_p, slot_b, version_info) -> - evaluate_version_info slot_p slot_b version_info - | Error e -> - ErrorGettingVersionInfo (Printexc.to_string e) - ) |> set - - | ErrorGettingVersionInfo msg -> - (* handle error while getting version information *) - let%lwt () = - Logs_lwt.err ~src:log_src - (fun m -> m "failed to get version information: %s" msg) - in - (* sleep and retry *) - let%lwt () = sleep_error_backoff () in - set GettingVersionInfo + let system_status = evaluate_version_info slot_p slot_b version_info in + let next_proc_state = match system_status with + | NeedsUpdate -> Downloading (Semver.to_string version_info.latest) + | _ -> Sleeping (config.check_for_updates_interval) + in + return { + process_state = next_proc_state; + version_info = Some version_info; + system_status = system_status + } + | Error exn -> + let exn_str = Printexc.to_string exn in + let%lwt () = Logs_lwt.err ~src:log_src + (fun m -> m "failed to get version information: %s" exn_str) + in + return { + process_state = Sleeping (config.error_backoff_duration); + (* unsetting version_info to indicate we are unclear about + current system state *) + version_info = None; + system_status = UpdateError (ErrorGettingVersionInfo exn_str); + } + ) + + | Sleeping duration -> + let%lwt () = Lwt_unix.sleep duration in + return {state with process_state = GettingVersionInfo} | Downloading version -> (* download latest version *) (match%lwt Lwt_result.catch (fun () -> ClientI.download version) with | Ok bundle_path -> - Installing bundle_path - |> set + return {state with process_state = Installing bundle_path} | Error exn -> - ErrorDownloading (Printexc.to_string exn) - |> set + let exn_str = Printexc.to_string exn in + let%lwt () = Logs_lwt.err ~src:log_src + (fun m -> m "failed to download RAUC bundle: %s" exn_str) + in + return { state with + process_state = Sleeping config.error_backoff_duration; + system_status = UpdateError (ErrorDownloading exn_str); + } ) - | ErrorDownloading msg -> - (* handle error while downloading bundle *) - let%lwt () = - Logs_lwt.err ~src:log_src - (fun m -> m "failed to download RAUC bundle: %s" msg) - in - (* sleep and retry *) - let%lwt () = sleep_error_backoff () in - set GettingVersionInfo - | Installing bundle_path -> (* install bundle via RAUC *) (match%lwt Lwt_result.catch (fun () -> RaucI.install bundle_path) with | Ok () -> - let%lwt () = - Logs_lwt.info (fun m -> m "succesfully installed update (%s)" bundle_path) + let%lwt () = Logs_lwt.info + (fun m -> m "succesfully installed update (%s)" bundle_path) in - RebootRequired - |> set + return { state with + (* unsetting version_info, because it is now stale *) + version_info = None; + (* going back to GettingVersionInfo to update version_info *) + process_state = GettingVersionInfo; + } | Error exn -> let () = try Sys.remove bundle_path with | _ -> () in - ErrorInstalling (Printexc.to_string exn) - |> set + let exn_str = Printexc.to_string exn in + let%lwt () = + Logs_lwt.err ~src:log_src + (fun m -> m "failed to install RAUC bundle: %s" exn_str) + in + return { state with + process_state = Sleeping config.error_backoff_duration; + system_status = UpdateError (ErrorInstalling exn_str); + } ) - | ErrorInstalling msg -> - (* handle installation error *) - let%lwt () = - Logs_lwt.err ~src:log_src - (fun m -> m "failed to install RAUC bundle: %s" msg) - in - (* sleep and retry *) - let%lwt () = sleep_error_backoff () in - set GettingVersionInfo - - | UpToDate _ - | RebootRequired - | OutOfDateVersionSelected - | ReinstallRequired -> - (* sleep and recheck for new updates *) - let%lwt () = sleep_update_check () in - set GettingVersionInfo - - (** Finite state machine handling updates *) - let rec run ~set_state state = + let rec run_rec set_state state = let%lwt next_state = run_step state in set_state next_state; - run ~set_state next_state + run_rec set_state next_state + + let run set_state = run_rec set_state initial_state end let default_config : config = { @@ -255,7 +272,6 @@ let build_deps ~connman ~(rauc : Rauc.t) : Lwt.return (module Deps : ServiceDeps) let start ~connman ~(rauc : Rauc.t) = - let initial_state = GettingVersionInfo in let state_s, set_state = Lwt_react.S.create initial_state in let () = Logs.info ~src:log_src (fun m -> m "update URL: %s" Config.System.update_url) in @@ -263,7 +279,7 @@ let start ~connman ~(rauc : Rauc.t) = let service = begin let%lwt deps = build_deps ~connman ~rauc in let module UpdateServiceI = Make(val deps) in - UpdateServiceI.run ~set_state initial_state + UpdateServiceI.run set_state end in let () = Logs.info ~src:log_src (fun m -> m "Started") in diff --git a/controller/server/update.mli b/controller/server/update.mli index f5274133..80e0479e 100644 --- a/controller/server/update.mli +++ b/controller/server/update.mli @@ -1,5 +1,4 @@ -(** Type containing version information. -*) +(** Type containing version information. *) type version_info = {(* the latest available version *) latest : Semver.t @@ -12,21 +11,38 @@ type version_info = } [@@deriving sexp_of] -(** State of update mechanism *) -type state = - | GettingVersionInfo +type update_error = | ErrorGettingVersionInfo of string - | UpToDate of version_info - | Downloading of string | ErrorDownloading of string - | Installing of string | ErrorInstalling of string +[@@deriving sexp_of] + +type system_status = + | UpToDate + | NeedsUpdate | RebootRequired | OutOfDateVersionSelected | ReinstallRequired + | UpdateError of update_error [@@deriving sexp_of] type sleep_duration = float (* seconds *) +[@@deriving sexp_of] + +(** State of update mechanism *) +type process_state = + | Sleeping of sleep_duration + | GettingVersionInfo + | Downloading of string + | Installing of string +[@@deriving sexp_of] + +type state = { + version_info: version_info option; + system_status: system_status; + process_state: process_state +} +[@@deriving sexp_of] type config = { (* time to sleep in seconds until retrying after a (Curl/HTTP) error *) @@ -42,9 +58,13 @@ module type ServiceDeps = sig val config : config end +(* exposed for unit testing purposes *) +val initial_state : state + module type UpdateService = sig - val run : set_state:(state -> unit) -> state -> unit Lwt.t + val run : (state -> unit) -> unit Lwt.t + (* exposed for unit testing purposes *) val run_step : state -> state Lwt.t end diff --git a/controller/server/view/status_page.ml b/controller/server/view/status_page.ml index d66b7b30..03b63ed5 100644 --- a/controller/server/view/status_page.ml +++ b/controller/server/view/status_page.ml @@ -92,21 +92,26 @@ let other_slot = let open Rauc.Slot in function let suggested_action_of_state (update:Update.state) (rauc:rauc_state) booted_slot = let target_slot = other_slot booted_slot in - match (update, rauc) with + match (update.system_status, rauc) with | (RebootRequired, _) -> Some (Definition.description reboot_call) | (OutOfDateVersionSelected, Status _) -> Some (Definition.description ( switch_to_newer_system_call target_slot )) - | (UpToDate {booted; inactive}, Status _) when booted <> inactive -> - Some (Definition.description ( - switch_to_older_system_call target_slot - )) | (ReinstallRequired, _) -> Some (Definition.description ( reinstall_call target_slot )) + | (UpToDate, Status _) -> + Option.bind update.version_info (fun {booted; inactive; _} -> + if (booted <> inactive) then + Some (Definition.description ( + switch_to_older_system_call target_slot + )) + else + None + ) | _ -> None diff --git a/controller/tests/server/update/outcome.ml b/controller/tests/server/update/outcome.ml index acf61354..92ee48a9 100644 --- a/controller/tests/server/update/outcome.ml +++ b/controller/tests/server/update/outcome.ml @@ -25,20 +25,21 @@ let slot_spec_to_outcome ({booted_slot; primary_slot; input_versions} : Helpers. (* Checks if the state returned by UpdateService matches the expected outcome as determined by [slot_spec_to_outcome] *) let state_matches_expected_outcome state outcome = - match (outcome, state) with - | (InstallVsn v1, Update.Downloading v2) -> - (Semver.to_string v1) = v2 - | (InstallVsn _, _) -> false - | (DoNothingOrProduceWarning, Update.ErrorGettingVersionInfo _) -> true - | (DoNothingOrProduceWarning, Update.UpToDate _) -> true - | (DoNothingOrProduceWarning, Update.OutOfDateVersionSelected) -> true - | (DoNothingOrProduceWarning, Update.RebootRequired) -> true - | (DoNothingOrProduceWarning, Update.ReinstallRequired) -> true - (* should not _directly_ return to GettingVersionInfo state *) - | (DoNothingOrProduceWarning, Update.GettingVersionInfo) -> false - (* all the other states are part of the installation process - and are treated as errors *) - | (DoNothingOrProduceWarning, _) -> false + let open Update in + match (outcome, state.system_status, state.process_state) with + | (InstallVsn v1, NeedsUpdate, Downloading v2) -> + (Semver.to_string v1) = v2 && + Option.fold ~none:false ~some:(fun v -> v.latest = v1) state.version_info + | (InstallVsn _, _, _) -> false + | (DoNothingOrProduceWarning, UpdateError (ErrorGettingVersionInfo _), Sleeping _) -> true + | (DoNothingOrProduceWarning, UpToDate, Sleeping _) -> true + | (DoNothingOrProduceWarning, OutOfDateVersionSelected, Sleeping _) -> true + | (DoNothingOrProduceWarning, RebootRequired, Sleeping _) -> true + | (DoNothingOrProduceWarning, ReinstallRequired, Sleeping _) ->true + (* should not _directly_ return to GettingVersionInfo state *) (*TODO: redundant now *) + | (DoNothingOrProduceWarning, _, Update.GettingVersionInfo) -> false + (* all the other state combos are treated as errors *) + | (DoNothingOrProduceWarning, _, _) ->false (** Tests if the input UpdateService run with the given [Helpers.system_slot_spec] [case] scenario produces the expected outcome state (defined by @@ -64,7 +65,7 @@ let test_slot_spec case = let () = Helpers.setup_mocks_from_system_slot_spec mocks case in let module UpdateServiceI = (val mocks.update_service) in - let%lwt out_state = UpdateServiceI.run_step GettingVersionInfo in + let%lwt out_state = UpdateServiceI.run_step Update.initial_state in if state_matches_expected_outcome out_state expected_outcome then Lwt.return () else diff --git a/controller/tests/server/update/scenario.ml b/controller/tests/server/update/scenario.ml index 859e2349..acd6f422 100644 --- a/controller/tests/server/update/scenario.ml +++ b/controller/tests/server/update/scenario.ml @@ -35,7 +35,6 @@ let str_match_with_magic_pat expected actual = ) exp_parts in string_match exp_regexp actual 0 -let state_formatter out inp = Format.fprintf out "%s" (Helpers.statefmt inp) let testable_state = let state_to_str s = @@ -43,8 +42,12 @@ let testable_state = (* using _hum instead of _mach, because _mach seems to remove whitespace between atoms in some cases *) |> Sexplib.Sexp.to_string_hum - (* remove new lines to ignore whitespace differences *) + (* ignore whitespace differences *) |> Str.global_replace (Str.regexp_string "\n") "" + |> Str.global_replace (Str.regexp "[ ]+") " " + in + let state_formatter out inp = + Format.fprintf out "%s" (state_to_str inp) in let state_eq expected actual = (expected == actual) || ( @@ -119,15 +122,20 @@ let rec run_test_scenario (test_context: Helpers.test_context) expected_state_se else Lwt.return () (* NOTE: this is almost the same as the `Outcome.test_slot_spec, - except that it expects a specific state outcome and uses the + except that it expects only a system state outcome and uses the `run_test_scenario` machinery. *) let scenario_from_system_spec ?(booted_slot=Rauc.Slot.SystemA) ?(primary_slot=(Some Rauc.Slot.SystemA)) ~(input_versions:Update.version_info) - (expected_state:Update.state) = + (expected_system_status:Update.system_status) = - let init_state = Update.GettingVersionInfo in + let init_state = Update.initial_state in + let expected_state: Update.state = { + version_info = Some input_versions; + system_status = expected_system_status; + process_state = Sleeping Helpers.default_test_config.check_for_updates_interval; + } in fun mocks -> let expected_state_sequence = @@ -139,7 +147,7 @@ let scenario_from_system_spec input_versions = input_versions } ); - StateReached Update.GettingVersionInfo; + StateReached Update.initial_state; StateReached expected_state; ] in diff --git a/controller/tests/server/update/update_prop_tests.ml b/controller/tests/server/update/update_prop_tests.ml index bea2df25..7d6f9b37 100644 --- a/controller/tests/server/update/update_prop_tests.ml +++ b/controller/tests/server/update/update_prop_tests.ml @@ -98,8 +98,8 @@ let test_random_failure_case = (Printexc.get_backtrace ()) (state_seq_to_str state_seq) - | (Ok Update.GettingVersionInfo) -> - Queue.push Update.GettingVersionInfo state_seq; + | Ok ({process_state = Update.GettingVersionInfo; _} as state) -> + Queue.push state state_seq; true | (Ok state) -> if (c < loop_lim) then @@ -112,7 +112,7 @@ let test_random_failure_case = loop_lim (state_seq_to_str state_seq) in - do_while 5 Update.GettingVersionInfo + do_while 5 Update.initial_state in QCheck2.Test.make ~count:10_000 diff --git a/controller/tests/server/update/update_tests.ml b/controller/tests/server/update/update_tests.ml index 0c9995e8..ee34ef4f 100644 --- a/controller/tests/server/update/update_tests.ml +++ b/controller/tests/server/update/update_tests.ml @@ -4,15 +4,24 @@ open Update_test_helpers (* Main test scenario: full update process *) let both_out_of_date ({update_client; rauc}: Helpers.test_context) = - let init_state = GettingVersionInfo in let booted_version = "10.0.0" in let inactive_version = "9.0.0" in let upstream_version = "10.0.2" in - + let vsn_info = { + booted = Semver.of_string booted_version |> Option.get; + inactive = Semver.of_string inactive_version |> Option.get; + latest = Semver.of_string upstream_version |> Option.get; + } in let expected_bundle_name vsn = Mock_update_client.test_bundle_name ^ Scenario._WILDCARD_PAT ^ vsn ^ Scenario._WILDCARD_PAT in + let base_expected_state = { + version_info = Some vsn_info; + system_status = NeedsUpdate; + process_state = GettingVersionInfo + } in + let expected_state_sequence = [ Scenario.UpdateMock (fun () -> @@ -24,9 +33,11 @@ let both_out_of_date ({update_client; rauc}: Helpers.test_context) = ("BUNDLE_CONTENTS: " ^ upstream_version); update_client#set_latest_version upstream_version; ); - Scenario.StateReached GettingVersionInfo; - Scenario.StateReached (Downloading upstream_version); - Scenario.StateReached (Installing (Scenario._WILDCARD_PAT ^ expected_bundle_name upstream_version)); + Scenario.StateReached Update.initial_state; + Scenario.StateReached {base_expected_state with process_state = Downloading upstream_version}; + Scenario.StateReached {base_expected_state with + process_state = (Installing (Scenario._WILDCARD_PAT ^ expected_bundle_name upstream_version)); + }; Scenario.ActionDone ( "bundle was installed into secondary slot", fun _ -> @@ -37,17 +48,34 @@ let both_out_of_date ({update_client; rauc}: Helpers.test_context) = upstream_version status.version in Lwt.return true ); - Scenario.StateReached RebootRequired; - Scenario.StateReached GettingVersionInfo; + Scenario.StateReached {base_expected_state with + version_info = None; + process_state = GettingVersionInfo; + }; + Scenario.StateReached { + version_info = Some {vsn_info with inactive = vsn_info.latest }; + system_status = RebootRequired; + process_state = Sleeping Helpers.default_test_config.check_for_updates_interval; + }; ] in - (expected_state_sequence, init_state) + (expected_state_sequence, Update.initial_state) let delete_downloaded_bundle_on_err ({update_client; rauc}: Helpers.test_context) = let inactive_version = "9.0.0" in + let booted_version = inactive_version in let upstream_version = "10.0.0" in - let init_state = Downloading upstream_version in + let vsn_info = { + booted = Semver.of_string booted_version |> Option.get; + inactive = Semver.of_string inactive_version |> Option.get; + latest = Semver.of_string upstream_version |> Option.get; + } in + let init_state = { + system_status = NeedsUpdate; + version_info = Some vsn_info; + process_state = Downloading upstream_version; + } in let expected_bundle_name vsn = Mock_update_client.test_bundle_name ^ Scenario._WILDCARD_PAT ^ vsn ^ Scenario._WILDCARD_PAT in @@ -61,11 +89,13 @@ let delete_downloaded_bundle_on_err ({update_client; rauc}: Helpers.test_context as invalid by mock RAUC *) update_client#add_bundle upstream_version "CORRUPT_BUNDLE_CONTENTS" ); - Scenario.StateReached (Downloading upstream_version); - Scenario.StateReached (Installing (Scenario._WILDCARD_PAT ^ expected_bundle_name upstream_version)); + Scenario.StateReached init_state; + Scenario.StateReached {init_state with process_state = + (Installing (Scenario._WILDCARD_PAT ^ expected_bundle_name upstream_version)); + }; Scenario.ActionDone ( "bundle was deleted from path due to installation error", - fun (Installing path) -> + fun ({process_state = Installing path; _}) -> let status = rauc#get_slot_status SystemB in Alcotest.(check string) "Inactive slot remains in the same version" @@ -74,54 +104,60 @@ let delete_downloaded_bundle_on_err ({update_client; rauc}: Helpers.test_context "Downloaded corrupt bundle was deleted" false (Sys.file_exists path); Lwt.return true ); - Scenario.StateReached (ErrorInstalling Scenario._WILDCARD_PAT); - Scenario.StateReached GettingVersionInfo; + Scenario.StateReached {init_state with + process_state = Sleeping Helpers.default_test_config.error_backoff_duration; + system_status = UpdateError (ErrorInstalling Scenario._WILDCARD_PAT); + }; + Scenario.StateReached {init_state with + process_state = GettingVersionInfo; + system_status = UpdateError (ErrorInstalling Scenario._WILDCARD_PAT); + }; ] in (expected_state_sequence, init_state) -let sleep_after_error_or_check_test () = - (* long-ish timeouts, but these will run in parallel, so no biggie *) - let test_config = { - error_backoff_duration = 1.0; - check_for_updates_interval = 2.0; - } in - let ({update_service; _}: Helpers.test_context) = Helpers.init_test_deps ~test_config () in +let sleep_on_get_version_err _ () = + let always_fail_gen () = Lwt.return true in + let { update_service ; _}: Helpers.test_context = Helpers.init_test_deps + ~failure_gen_upd:always_fail_gen () + in let module UpdateServiceI = (val update_service) in + let init_state = Update.initial_state in + let expected_state = { + version_info = None; + system_status = UpdateError (ErrorGettingVersionInfo Scenario._WILDCARD_PAT); + process_state = Sleeping Helpers.default_test_config.error_backoff_duration; + } in + let%lwt out_state = UpdateServiceI.run_step init_state in + Lwt.return @@ Alcotest.check Scenario.testable_state + "Output state matches" + expected_state + out_state - let error_states = [ - ErrorGettingVersionInfo "err"; - ErrorInstalling "err"; - ErrorDownloading "err"; - ] in - let post_check_states = [ - UpToDate (Helpers.vsn_triple_to_version_info (Helpers.v1, Helpers.v1, Helpers.v1)); - RebootRequired; - OutOfDateVersionSelected; - ReinstallRequired; - ] in - - let test_state expected_timeout inp_state = - let start_time = Unix.gettimeofday () in - (* NOTE: running the same step TWICE to ensure - that we execute the code in the same thread multiple times *) - let%lwt _ = UpdateServiceI.run_step inp_state in - let%lwt _ = UpdateServiceI.run_step inp_state in - let end_time = Unix.gettimeofday () in - let elasped_seconds = end_time -. start_time in - if elasped_seconds > (expected_timeout *. 2.0) then - Lwt.return () - else - Lwt.return @@ Alcotest.fail @@ - Format.sprintf "Slept shorter than expected (expected %f; slept %f) after state %s" - (expected_timeout *. 2.0) elasped_seconds (Helpers.statefmt inp_state) - in - Lwt.join @@ - (List.map (test_state test_config.error_backoff_duration) error_states) - @ - (List.map (test_state test_config.check_for_updates_interval) post_check_states) +let sleep_on_download_err _ () = + let always_fail_gen () = Lwt.return true in + let { update_service ; _}: Helpers.test_context = Helpers.init_test_deps + ~failure_gen_upd:always_fail_gen () + in + let module UpdateServiceI = (val update_service) in + let init_state: Update.state = { + version_info = Some + { latest = Helpers.v2; booted = Helpers.v1; inactive = Helpers.v1; }; + system_status = NeedsUpdate; + process_state = Downloading (Semver.to_string Helpers.v2); + } in + let expected_state = { + version_info = None; + system_status = UpdateError (ErrorDownloading Scenario._WILDCARD_PAT); + process_state = Sleeping Helpers.default_test_config.error_backoff_duration; + } in + let%lwt out_state = UpdateServiceI.run_step init_state in + Lwt.return @@ Alcotest.check Scenario.testable_state + "Output state matches" + expected_state + out_state let both_newer_than_upstream = let input_versions = { @@ -129,9 +165,7 @@ let both_newer_than_upstream = inactive = Helpers.v2; latest = Helpers.v1; } in - let expected_state = - UpToDate input_versions - in + let expected_state = UpToDate in Scenario.scenario_from_system_spec ~input_versions expected_state let booted_newer_secondary_older = @@ -140,9 +174,7 @@ let booted_newer_secondary_older = booted = Helpers.v3; inactive = Helpers.v1; } in - let expected_state = - UpToDate input_versions - in + let expected_state = UpToDate in Scenario.scenario_from_system_spec ~input_versions expected_state let booted_older_secondary_newer = @@ -151,9 +183,7 @@ let booted_older_secondary_newer = booted = Helpers.v1; inactive = Helpers.v3; } in - let expected_state = - OutOfDateVersionSelected - in + let expected_state = OutOfDateVersionSelected in Scenario.scenario_from_system_spec ~input_versions expected_state let booted_current_secondary_current = @@ -162,9 +192,7 @@ let booted_current_secondary_current = booted = Helpers.v2; inactive = Helpers.v2; } in - let expected_state = - UpToDate input_versions - in + let expected_state = UpToDate in Scenario.scenario_from_system_spec ~input_versions expected_state let booted_current_secondary_older = @@ -173,9 +201,7 @@ let booted_current_secondary_older = booted = Helpers.v2; inactive = Helpers.v1; } in - let expected_state = - UpToDate input_versions - in + let expected_state = UpToDate in Scenario.scenario_from_system_spec ~input_versions expected_state let booted_older_secondary_current = @@ -184,8 +210,7 @@ let booted_older_secondary_current = booted = Helpers.v1; inactive = Helpers.v2; } in - let expected_state = OutOfDateVersionSelected - in + let expected_state = OutOfDateVersionSelected in Scenario.scenario_from_system_spec ~input_versions expected_state let () = @@ -219,9 +244,10 @@ let () = [ Alcotest_lwt.test_case "Delete downloaded bundle on install error" `Quick (fun _ () -> Scenario.run delete_downloaded_bundle_on_err); - - Alcotest_lwt.test_case "Sleep for a duration after error or check" - `Quick (fun _ () -> sleep_after_error_or_check_test ()); + Alcotest_lwt.test_case "Update enters sleep after get version error" + `Quick sleep_on_get_version_err; + Alcotest_lwt.test_case "Update enters sleep after get download error" + `Quick sleep_on_get_version_err; ] ); ( "All version/slot combinations", List.map Outcome.test_slot_spec Helpers.all_possible_slot_spec_combos ); From 395b806d9885e7ed65184ec5cc6410bec1fa815f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignas=20Vy=C5=A1niauskas?= Date: Thu, 28 Nov 2024 09:45:02 +0200 Subject: [PATCH 2/3] Formatting --- controller/tests/server/update/outcome.ml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controller/tests/server/update/outcome.ml b/controller/tests/server/update/outcome.ml index 92ee48a9..8d3ac413 100644 --- a/controller/tests/server/update/outcome.ml +++ b/controller/tests/server/update/outcome.ml @@ -35,11 +35,11 @@ let state_matches_expected_outcome state outcome = | (DoNothingOrProduceWarning, UpToDate, Sleeping _) -> true | (DoNothingOrProduceWarning, OutOfDateVersionSelected, Sleeping _) -> true | (DoNothingOrProduceWarning, RebootRequired, Sleeping _) -> true - | (DoNothingOrProduceWarning, ReinstallRequired, Sleeping _) ->true + | (DoNothingOrProduceWarning, ReinstallRequired, Sleeping _) -> true (* should not _directly_ return to GettingVersionInfo state *) (*TODO: redundant now *) | (DoNothingOrProduceWarning, _, Update.GettingVersionInfo) -> false (* all the other state combos are treated as errors *) - | (DoNothingOrProduceWarning, _, _) ->false + | (DoNothingOrProduceWarning, _, _) -> false (** Tests if the input UpdateService run with the given [Helpers.system_slot_spec] [case] scenario produces the expected outcome state (defined by From 1fc88d5c070de2e050f68602118eba1f39ab80f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignas=20Vy=C5=A1niauskas?= Date: Thu, 28 Nov 2024 09:46:52 +0200 Subject: [PATCH 3/3] Remove redundant case in outcome spec --- controller/tests/server/update/outcome.ml | 2 -- 1 file changed, 2 deletions(-) diff --git a/controller/tests/server/update/outcome.ml b/controller/tests/server/update/outcome.ml index 8d3ac413..6ef48c54 100644 --- a/controller/tests/server/update/outcome.ml +++ b/controller/tests/server/update/outcome.ml @@ -36,8 +36,6 @@ let state_matches_expected_outcome state outcome = | (DoNothingOrProduceWarning, OutOfDateVersionSelected, Sleeping _) -> true | (DoNothingOrProduceWarning, RebootRequired, Sleeping _) -> true | (DoNothingOrProduceWarning, ReinstallRequired, Sleeping _) -> true - (* should not _directly_ return to GettingVersionInfo state *) (*TODO: redundant now *) - | (DoNothingOrProduceWarning, _, Update.GettingVersionInfo) -> false (* all the other state combos are treated as errors *) | (DoNothingOrProduceWarning, _, _) -> false