diff --git a/app/src/ActiveServices.tsx b/app/src/ActiveServices.tsx index 6e9ecf90..228024ad 100644 --- a/app/src/ActiveServices.tsx +++ b/app/src/ActiveServices.tsx @@ -48,7 +48,11 @@ export const ActiveServices = () => { {" / "} {svc.state === "RUNNING" ? ( { className="p-0 ms-2" title="Logs" onClick={() => - setLogTarget({ kind: "service", experimentId, serviceId: id, title: id }) + setLogTarget({ + kind: "service", + experimentId, + serviceId: id, + runId: svc.runId, + title: id, + }) } > @@ -79,7 +89,7 @@ export const ActiveServices = () => { variant="link" className="p-0 ms-2 text-danger" title="Stop" - onClick={() => client.service_stop(experimentId, id)} + onClick={() => client.service_stop(experimentId, id, svc.runId)} > diff --git a/app/src/App.tsx b/app/src/App.tsx index b5beae8b..8ea2330a 100644 --- a/app/src/App.tsx +++ b/app/src/App.tsx @@ -219,7 +219,9 @@ export default () => { {/* The run below the experiment, so that both have room */}
{isScoped(view) && } - {(view === "tasks" || view === "graph") && } + {(view === "tasks" || view === "graph" || view === "services") && ( + + )}
+ )} void }) => { ))} - {groups !== null && ( - <> - - - - )} { const navigate = useNavigate(); const route = useAppSelector((s) => s.db.route); const currentExperiment = useAppSelector((s) => s.db.currentExperiment); + const currentRunId = useAppSelector((s) => + currentExperiment ? s.db.experiments.byId[currentExperiment]?.current_run_id : null + ); const currentRun = useAppSelector((s) => s.db.currentRun); const runs = useAppSelector((s) => s.db.runs); - // Fetch the run history whenever the selected experiment changes + // Fetch the run history whenever the selected experiment or its current run changes useEffect(() => { if (currentExperiment) { client.request_runs(currentExperiment); } - }, [currentExperiment]); + }, [currentExperiment, currentRunId]); if (!currentExperiment || runs.length <= 1) { return null; diff --git a/app/src/Services.tsx b/app/src/Services.tsx index ca4e89c3..2dd9545c 100644 --- a/app/src/Services.tsx +++ b/app/src/Services.tsx @@ -17,18 +17,27 @@ const STATE_COLOR: { [key: string]: string } = { const Services = () => { const services = useAppSelector((state) => state.db.services); const currentExperiment = useAppSelector((state) => state.db.currentExperiment); + const currentRun = useAppSelector((state) => state.db.currentRun); + const exp = useAppSelector((state) => + currentExperiment ? state.db.experiments.byId[currentExperiment] : null + ); + const currentRunId = exp?.current_run_id; + const effectiveRunId = currentRun || currentRunId || null; const connectionStatus = useAppSelector((state) => state.db.connectionStatus); const [logTarget, setLogTarget] = useState(); - // Only the services of the selected experiment are requested + // Only the services of the selected experiment and run are requested useEffect(() => { if (connectionStatus === "connected" && currentExperiment !== null) - client.request_services(currentExperiment); - }, [connectionStatus, currentExperiment]); + client.request_services(currentExperiment, currentRun); + }, [connectionStatus, currentExperiment, currentRun, currentRunId]); - const keys = services.ids.filter( - (key) => services.byId[key].experimentId === currentExperiment, - ); + const keys = services.ids.filter((key) => { + const svc = services.byId[key]; + if (svc.experimentId !== currentExperiment) return false; + if (effectiveRunId && svc.runId && svc.runId !== effectiveRunId) return false; + return true; + }); return (
@@ -81,7 +90,9 @@ const Services = () => { className="me-1" href={`/services/${encodeURIComponent( experimentId, - )}/${encodeURIComponent(id)}/`} + )}/${encodeURIComponent(id)}/${ + svc.runId ? `?runId=${encodeURIComponent(svc.runId)}` : "" + }`} target="_blank" > Open @@ -91,7 +102,7 @@ const Services = () => { size="sm" variant="outline-danger" className="me-1" - onClick={() => client.service_stop(experimentId, id)} + onClick={() => client.service_stop(experimentId, id, svc.runId)} > Stop @@ -101,7 +112,7 @@ const Services = () => { size="sm" variant="outline-success" className="me-1" - onClick={() => client.service_start(experimentId, id)} + onClick={() => client.service_start(experimentId, id, svc.runId)} > Start @@ -114,6 +125,7 @@ const Services = () => { kind: "service", experimentId, serviceId: id, + runId: svc.runId, title: id, }) } diff --git a/app/src/Tasks.tsx b/app/src/Tasks.tsx index e4e5f912..95ea4353 100644 --- a/app/src/Tasks.tsx +++ b/app/src/Tasks.tsx @@ -10,6 +10,7 @@ import { useAppSelector, useNavigate } from "./store"; import { useShortcuts } from "./shortcuts"; import FilterInput from "./FilterInput"; import { KeyHints, SortHeader, SortLabel, sortedMessage } from "./ui/sorting"; +import { CollapseExpandIcon } from "./ui/icons"; type SortKey = "status" | "task" | "date" | "name"; @@ -332,6 +333,18 @@ export default () => { setCollapsed(new Set()); } + const allCollapsed = + groups !== null && groups.length > 0 && groups.every((g) => collapsed.has(g.key)); + + function toggleAllGroups() { + if (groups === null) return; + if (allCollapsed) { + setCollapsed(new Set()); + } else { + setCollapsed(new Set(groups.map((g) => g.key))); + } + } + // --- Job actions function cancelConfirm() { @@ -374,6 +387,21 @@ export default () => { flexWrap: "wrap", }} > + {groups !== null && ( + + )} Filter { )} - {groups !== null && ( - <> - - - - )}
{trimmedQuery && filterResult.error !== null ? (
{filterResult.error}
diff --git a/app/src/client.ts b/app/src/client.ts index 70960d43..2c5b04ad 100644 --- a/app/src/client.ts +++ b/app/src/client.ts @@ -360,8 +360,10 @@ class Client { } // Services of an experiment (shown by the services page) - request_services(experimentId: string): void { - this.send("services", { experimentId }); + request_services(experimentId: string, runId?: string | null): void { + const payload: any = { experimentId }; + if (runId) payload.runId = runId; + this.send("services", payload); } request_runs(experimentId: string): void { @@ -415,12 +417,16 @@ class Client { this.send("orphans.delete"); } - service_start(experimentId: string, id: string): void { - this.send("service.start", { experimentId, id }); + service_start(experimentId: string, id: string, runId?: string | null): void { + const payload: any = { experimentId, id }; + if (runId) payload.runId = runId; + this.send("service.start", payload); } - service_stop(experimentId: string, id: string): void { - this.send("service.stop", { experimentId, id }); + service_stop(experimentId: string, id: string, runId?: string | null): void { + const payload: any = { experimentId, id }; + if (runId) payload.runId = runId; + this.send("service.stop", payload); } quit(): void { diff --git a/app/src/reducers.ts b/app/src/reducers.ts index 84513603..98e6b0d9 100644 --- a/app/src/reducers.ts +++ b/app/src/reducers.ts @@ -1,4 +1,4 @@ -import { combineReducers, createSlice, PayloadAction } from "@reduxjs/toolkit"; +import { combineReducers, createSlice, Draft, PayloadAction } from "@reduxjs/toolkit"; import { messageSlice } from "./ui/messages"; import _ from "lodash"; import { Route, currentLocationRoute, isScoped, route } from "./routes"; @@ -136,10 +136,11 @@ export type OrphanJob = Job & { type ServiceStatus = "STOPPED" | "STARTING" | "RUNNING" | "STOPPING" | "ERROR"; export type ServiceInformation = { - // Unique key ("/"): service ids are only unique within - // an experiment + // Unique key ("//" or "/"): service ids are only unique within + // an experiment and run key: string; experimentId: string; + runId?: string | null; id: string; description: string; state: ServiceStatus; @@ -238,6 +239,29 @@ export type State = { const initialRoute = currentLocationRoute(); const initialScope = isScoped(initialRoute.view) ? initialRoute : route("experiments"); +/** + * Drops the jobs when the current run of the shown experiment changes (a + * relaunch) while the current run is shown: they are requested again for the + * new run (App.tsx) + */ +function dropJobsOnRunChange( + draft: Draft, + experimentId: string, + prevRunId: string | null | undefined, + runId: string | null | undefined, +) { + if ( + draft.currentExperiment === experimentId && + draft.currentRun === null && + runId && + prevRunId && + prevRunId !== runId + ) { + draft.jobs = { byId: {}, ids: [] }; + draft.jobsRequested = []; + } +} + export const slice = createSlice({ name: "db", initialState: { @@ -326,11 +350,14 @@ export const slice = createSlice({ addExperiment(draft, action: PayloadAction) { const exp = action.payload; + const prevRunId = draft.experiments.byId[exp.experiment_id]?.current_run_id; if (draft.experiments.byId[exp.experiment_id] === undefined) { draft.experiments.ids.push(exp.experiment_id); } draft.experiments.byId[exp.experiment_id] = exp; + dropJobsOnRunChange(draft, exp.experiment_id, prevRunId, exp.current_run_id); + // Set as current experiment if it's the first one, unless the route // names the experiment(s) shown if (draft.experiments.ids.length === 1 && !isScoped(draft.route.view)) { @@ -349,7 +376,9 @@ export const slice = createSlice({ ) { const exp = action.payload; if (draft.experiments.byId[exp.experiment_id] !== undefined) { + const prevRunId = draft.experiments.byId[exp.experiment_id].current_run_id; _.merge(draft.experiments.byId[exp.experiment_id], exp); + dropJobsOnRunChange(draft, exp.experiment_id, prevRunId, exp.current_run_id); } }, diff --git a/app/src/ui/icons.tsx b/app/src/ui/icons.tsx new file mode 100644 index 00000000..62b7ee60 --- /dev/null +++ b/app/src/ui/icons.tsx @@ -0,0 +1,70 @@ +import React from "react"; + +interface CollapseExpandIconProps { + collapsed: boolean; + className?: string; +} + +/** + * Vertical >< (collapse) / <> (expand) icon. + * + * - When collapsed is true (action is expand all): + * renders vertical <> (chevrons pointing outward: ^ over v). + * - When collapsed is false (action is collapse all): + * renders vertical >< (chevrons pointing inward: v over ^). + */ +export const CollapseExpandIcon: React.FC = ({ + collapsed, + className, +}) => ( + +); diff --git a/docs/source/interfaces.md b/docs/source/interfaces.md index f2697e17..14429e2b 100644 --- a/docs/source/interfaces.md +++ b/docs/source/interfaces.md @@ -88,8 +88,10 @@ Then open `http://localhost:12345` in your browser. - **Orphan & Stray Jobs**: Jobs no experiment references, with their disk usage; kill or delete them one by one, or all at once (with confirmation) - **Services**: The services of the selected experiment (of all experiments by - default), also when the experiment is not running; start, stop and open - them (through `/services///`), or view their logs + default) and run (the current one by default, or the one chosen with the + run selector), also when the experiment is not running; start, stop and + open them (through `/services///`, with `?runId=` + for a service of a given run), or view their logs - **Active Services**: The services starting or running, of all the experiments, are listed under the navigation bar on every page (with their experiment), with links to open them, see their logs or stop them diff --git a/src/experimaestro/scheduler/experiment.py b/src/experimaestro/scheduler/experiment.py index 714b3c74..63b74579 100644 --- a/src/experimaestro/scheduler/experiment.py +++ b/src/experimaestro/scheduler/experiment.py @@ -126,6 +126,8 @@ def on_service_added(self, service): state_dict = Service.serialize_state_dict(service.state_dict()) service_class = f"{service.__class__.__module__}.{service.__class__.__name__}" event = ServiceAddedEvent( + experiment_id=self.experiment_id, + run_id=self.run_id, service_id=service.id, description=service.description(), service_class=service_class, @@ -710,7 +712,12 @@ def _finalize_run(self, status: str) -> None: self._status = ExperimentStatus.FAILED # Write RunCompletedEvent before closing the event writer - event = RunCompletedEvent(status=status, ended_at=datetime.now().isoformat()) + event = RunCompletedEvent( + experiment_id=self.experiment_id, + run_id=self.run_id, + status=status, + ended_at=datetime.now().isoformat(), + ) self._event_writer.write_event(event) # Close the event writer to flush any buffered events diff --git a/src/experimaestro/scheduler/service_operations.py b/src/experimaestro/scheduler/service_operations.py index 16d7e058..a5335f53 100644 --- a/src/experimaestro/scheduler/service_operations.py +++ b/src/experimaestro/scheduler/service_operations.py @@ -24,11 +24,18 @@ def find_service( - provider: "StateProvider", experiment_id: str, service_id: str + provider: "StateProvider", + experiment_id: str, + service_id: str, + run_id: str | None = None, ) -> BaseService | None: - """Service of an experiment (current run), if any""" + """Service of an experiment (current run or specific run), if any""" return next( - (s for s in provider.get_services(experiment_id) if s.id == service_id), + ( + s + for s in provider.get_services(experiment_id, run_id=run_id) + if s.id == service_id + ), None, ) diff --git a/src/experimaestro/scheduler/state_provider.py b/src/experimaestro/scheduler/state_provider.py index fb25b80a..a4507250 100644 --- a/src/experimaestro/scheduler/state_provider.py +++ b/src/experimaestro/scheduler/state_provider.py @@ -22,12 +22,18 @@ from pathlib import Path from abc import ABC, abstractmethod from typing import Any, Optional, TYPE_CHECKING -from collections.abc import Callable +from collections.abc import Callable, Hashable if TYPE_CHECKING: from experimaestro.core.serialization import ExperimentInfo from experimaestro.logs.source import LogSource - from experimaestro.scheduler.state_status import ExperimentEventBase, WarningEvent + from experimaestro.scheduler.state_status import ( + ExperimentEventBase, + ServiceAddedEvent, + ServiceEventBase, + ServiceStateChangedEvent, + WarningEvent, + ) from experimaestro.scheduler.interfaces import ( BaseAction, @@ -1375,7 +1381,10 @@ def apply_event(self, event: "EventBase") -> None: cached experiments (job counts) - ActionAddedEvent: applied to the cached experiments, drops the cached objects of the run - - ExperimentUpdatedEvent: Invalidates experiment cache + - ExperimentUpdatedEvent / RunUpdatedEvent / RunCompletedEvent: + invalidate the experiment cache (the current run may change) + - ServiceStateChangedEvent: updates the cached services in place + - ServiceAddedEvent: drops the cached services lists of its run Cached queries are invalidated for the events that change *which* entities exist: state/progress/carbon events update the cached job @@ -1391,6 +1400,10 @@ def apply_event(self, event: "EventBase") -> None: JobProgressEvent, JobSubmittedEvent, ExperimentUpdatedEvent, + RunUpdatedEvent, + RunCompletedEvent, + ServiceAddedEvent, + ServiceStateChangedEvent, ) # Handle job state/progress/carbon events @@ -1419,15 +1432,56 @@ def apply_event(self, event: "EventBase") -> None: self._apply_to_cached_experiments(event) self._update_cached_queries(event) - # Handle experiment events - invalidate cache to force refresh - elif isinstance(event, ExperimentUpdatedEvent): - self._query_cache.invalidate(event.experiment_id) - with self._experiment_cache_lock: - keys_to_remove = [ - k for k in self._experiment_cache if k[0] == event.experiment_id - ] - for key in keys_to_remove: - del self._experiment_cache[key] + # Experiment/run events (a new run, a run that ended): which run is + # current and the runs themselves change, so the experiment is re-read + elif isinstance( + event, (ExperimentUpdatedEvent, RunUpdatedEvent, RunCompletedEvent) + ): + self._clear_experiment_cache(event.experiment_id) + + # Service events: the cached experiments are updated in place (state + # of their services); a new service changes the membership of the + # cached services lists of its run, which are dropped (and only them, + # to keep the started service instances of the other runs) + elif isinstance(event, ServiceStateChangedEvent): + self._apply_to_cached_experiments(event) + self._update_cached_service_state(event) + + elif isinstance(event, ServiceAddedEvent): + self._apply_to_cached_experiments(event) + self._drop_cached_services(event) + + def _cached_services_of( + self, event: "ServiceEventBase" + ) -> list[tuple[Hashable, list["BaseService"]]]: + """Cached services lists (key, services) of the run of a service event + + Events without a run id match every run of the experiment. + """ + return [ + (key, entry.value) + for key, entry in self._query_cache.entries() + if key[0] == "services" + and key[1] == event.experiment_id + and (not event.run_id or key[2] == event.run_id) + ] + + def _update_cached_service_state(self, event: "ServiceStateChangedEvent") -> None: + """Update the state of the cached (read-only) services in place + + Only the services read from storage follow the events: the services + started by this process manage their own state. + """ + for _, services in self._cached_services_of(event): + for service in services: + if service.id == event.service_id and isinstance(service, MockService): + service._state_name = event.state + + def _drop_cached_services(self, event: "ServiceAddedEvent") -> None: + """Drop the cached services lists that miss the added service""" + for key, services in self._cached_services_of(event): + if all(service.id != event.service_id for service in services): + self._query_cache.drop(key) def _apply_to_cached_experiments(self, event: "ExperimentEventBase") -> None: """Apply an experiment event to the cached runs of its experiment @@ -1642,8 +1696,8 @@ def get_services( all_services.extend(exp_services) return all_services - # Resolve run_id if needed - if run_id is None: + # Resolve run_id if needed (events without a run give an empty one) + if not run_id: run_id = self.get_current_run(experiment_id, refresh=refresh) if run_id is None: return [] diff --git a/src/experimaestro/scheduler/state_status.py b/src/experimaestro/scheduler/state_status.py index a157bc3a..27c1190a 100644 --- a/src/experimaestro/scheduler/state_status.py +++ b/src/experimaestro/scheduler/state_status.py @@ -502,6 +502,27 @@ class ConnectionState(str, Enum): DISCONNECTED = "disconnected" +#: The events that belong to a run (they have a ``run_id``) +RUN_EVENTS = ( + JobSubmittedEvent, + ExperimentJobStateEvent, + RunUpdatedEvent, + RunCompletedEvent, + ServiceAddedEvent, + ServiceStateChangedEvent, + ActionAddedEvent, + WarningEvent, + ErrorEvent, +) + + +def event_run_id(event: EventBase) -> str | None: + """Run of an event (None if not given, or for events outside runs)""" + if isinstance(event, RUN_EVENTS): + return event.run_id or None + return None + + @dataclass class ConnectionStateEvent(EventBase): """Event: The connection of the state provider changed (remote providers) @@ -2109,6 +2130,8 @@ def is_done(self) -> bool: __all__ = [ + "RUN_EVENTS", + "event_run_id", # Hardlink utilities "supports_hardlinks", "safe_link_or_copy", diff --git a/src/experimaestro/scheduler/workspace_state_provider.py b/src/experimaestro/scheduler/workspace_state_provider.py index 9fef4d66..0eabcc64 100644 --- a/src/experimaestro/scheduler/workspace_state_provider.py +++ b/src/experimaestro/scheduler/workspace_state_provider.py @@ -37,6 +37,7 @@ CarbonMetricsData, ) from experimaestro.scheduler.state_status import ( + event_run_id, EventBase, JobEventBase, ExperimentEventBase, @@ -716,6 +717,7 @@ def _on_experiment_events_created(self, experiment_id: str, events: list) -> boo "Discovered experiment: %s (%d events)", experiment_id, len(events) ) self._invalidate_known_experiment_ids() + self._clear_experiment_cache(experiment_id) # Get current run and pre-load experiment into cache run_id = self.get_current_run(experiment_id) @@ -744,9 +746,9 @@ def _on_experiment_events_created(self, experiment_id: str, events: list) -> boo if events: for event in events: self._notify_state_listeners(event) - self._notify_state_listeners( - ExperimentUpdatedEvent(experiment_id=experiment_id) - ) + self._notify_state_listeners( + ExperimentUpdatedEvent(experiment_id=experiment_id) + ) return True # Follow all experiments @@ -1144,6 +1146,14 @@ def _handle_experiment_event(self, experiment_id: str, event: EventBase) -> None ) run_id = self.get_current_run(experiment_id) + run_of_event = event_run_id(event) + if run_of_event and run_id and run_of_event != run_id: + self._clear_experiment_cache(experiment_id) + self._notify_state_listeners( + ExperimentUpdatedEvent(experiment_id=experiment_id) + ) + run_id = self.get_current_run(experiment_id) + if run_id is None: return diff --git a/src/experimaestro/tests/test_state_cache.py b/src/experimaestro/tests/test_state_cache.py index fb6ae657..b1d19ea5 100644 --- a/src/experimaestro/tests/test_state_cache.py +++ b/src/experimaestro/tests/test_state_cache.py @@ -12,7 +12,7 @@ import pytest from experimaestro.scheduler.query_cache import MISSING, QueryCache -from experimaestro.scheduler.state_provider import MockExperiment, MockJob +from experimaestro.scheduler.state_provider import MockExperiment, MockJob, MockService from experimaestro.scheduler.state_provider import OfflineStateProvider from experimaestro.scheduler.state_status import ( ActionAddedEvent, @@ -20,6 +20,9 @@ ExperimentUpdatedEvent, JobStateChangedEvent, JobTag, + RunUpdatedEvent, + ServiceAddedEvent, + ServiceStateChangedEvent, ) @@ -30,6 +33,8 @@ def __init__(self, experiment_ids=("exp1", "exp2")): super().__init__() self.calls: Counter = Counter() self.experiment_ids = list(experiment_ids) + # Stored services, by (experiment, run) + self.services: dict[tuple[str, str], list[MockService]] = {} def _make_job(self, experiment_id: str) -> MockJob: return MockJob( @@ -108,7 +113,7 @@ def _fetch_orphan_jobs(self, *, refresh=False, **kwargs): def _fetch_services_from_storage(self, experiment_id, run_id): self.calls[f"services:{experiment_id}"] += 1 - return [] + return list(self.services.get((experiment_id, run_id), [])) def _create_job(self, full_id, *args, **kwargs): return self._make_job(full_id) @@ -446,3 +451,79 @@ def test_datetime_arguments_are_hashable(provider): provider.get_experiments(since=since) provider.get_experiments(since=since) assert provider.calls["experiments"] == 1 + + +def test_current_run_invalidated_on_new_run_event(provider): + """When a new run event arrives, cached current_run is dropped""" + provider.get_current_run("exp1") + assert provider.calls["current_run:exp1"] == 1 + + provider.get_current_run("exp1") + assert provider.calls["current_run:exp1"] == 1 + + provider.apply_event(RunUpdatedEvent(experiment_id="exp1", run_id="run2")) + provider.get_current_run("exp1") + assert provider.calls["current_run:exp1"] == 2 + + +def test_service_added_event_invalidates_services_cache(provider): + """A new service drops the cached services of its run""" + provider.get_services("exp1", run_id="run1") + assert provider.calls["services:exp1"] == 1 + + provider.get_services("exp1", run_id="run1") + assert provider.calls["services:exp1"] == 1 + + provider.apply_event( + ServiceAddedEvent(experiment_id="exp1", run_id="run1", service_id="s1") + ) + provider.get_services("exp1", run_id="run1") + assert provider.calls["services:exp1"] == 2 + + +def _service(service_id: str, run_id: str) -> MockService: + return MockService( + service_id=service_id, + description_text="", + state_dict_data={}, + experiment_id="exp1", + run_id=run_id, + state="STOPPED", + ) + + +def test_service_added_event_keeps_other_runs(provider): + """A new service keeps the cached services of the other runs, and those + that already list it (the same instances are returned)""" + provider.services[("exp1", "run1")] = [_service("s1", "run1")] + provider.services[("exp1", "run2")] = [_service("s1", "run2")] + run1 = provider.get_services("exp1", run_id="run1") + run2 = provider.get_services("exp1", run_id="run2") + assert provider.calls["services:exp1"] == 2 + + # Already known (e.g. replayed event), or added to another run + provider.apply_event( + ServiceAddedEvent(experiment_id="exp1", run_id="run1", service_id="s1") + ) + provider.apply_event( + ServiceAddedEvent(experiment_id="exp1", run_id="run3", service_id="s2") + ) + assert provider.get_services("exp1", run_id="run1")[0] is run1[0] + assert provider.get_services("exp1", run_id="run2")[0] is run2[0] + assert provider.calls["services:exp1"] == 2 + + +def test_service_state_changed_updates_cached_services(provider): + """A service state change updates the cached service in place""" + provider.services[("exp1", "run1")] = [_service("s1", "run1")] + (service,) = provider.get_services("exp1", run_id="run1") + + provider.apply_event( + ServiceStateChangedEvent( + experiment_id="exp1", run_id="run1", service_id="s1", state="RUNNING" + ) + ) + (cached,) = provider.get_services("exp1", run_id="run1") + assert cached is service + assert cached.state.name == "RUNNING" + assert provider.calls["services:exp1"] == 1 diff --git a/src/experimaestro/tests/ui/drivers.py b/src/experimaestro/tests/ui/drivers.py index 21f158ab..a2dae297 100644 --- a/src/experimaestro/tests/ui/drivers.py +++ b/src/experimaestro/tests/ui/drivers.py @@ -260,8 +260,11 @@ async def delete_all_orphans(self) -> Notification | None: """Delete every finished orphan job (confirming if asked)""" ... - async def services(self, experiment_id: str) -> dict[str, str]: - """Services of an experiment, with their (lowercase) state""" + async def services( + self, experiment_id: str, run_id: str | None = None + ) -> dict[str, str]: + """Services of an experiment (its current run by default), with their + (lowercase) state""" ... async def graph( @@ -277,13 +280,13 @@ async def active_services(self) -> dict[str, str]: ... async def start_service( - self, experiment_id: str, service_id: str + self, experiment_id: str, service_id: str, run_id: str | None = None ) -> Notification | None: - """Start a service (None if not offered)""" + """Start a service (of the current run by default; None if not offered)""" ... async def stop_service( - self, experiment_id: str, service_id: str + self, experiment_id: str, service_id: str, run_id: str | None = None ) -> Notification | None: """Stop a service (None if not offered)""" ... diff --git a/src/experimaestro/tests/ui/scenario.py b/src/experimaestro/tests/ui/scenario.py index 8564a119..8d19916a 100644 --- a/src/experimaestro/tests/ui/scenario.py +++ b/src/experimaestro/tests/ui/scenario.py @@ -257,6 +257,11 @@ def _serve(self, running: threading.Event): "baseline": [("tensorboard", "TensorBoard (baseline)")], } +#: Services of former runs, by (experiment, run) +FORMER_RUN_SERVICES = { + ("baseline", "20260101_100000"): [("notebook", "Notebook (first run)")], +} + def stdout_text(job_id: str) -> str: return f"stdout of {job_id}\nsecond line\n" @@ -305,9 +310,12 @@ def __init__(self, workspace: Path): self.orphans.append(spec.identifier) (self.jobs[ORPHAN_DONE].path / "data.bin").write_bytes(b"x" * ORPHAN_DATA_SIZE) - for xp_id, services in SERVICES.items(): - run_id = self.runs[xp_id][-1].run_id - self.services[xp_id] = [ + runs_services = { + (xp_id, self.runs[xp_id][-1].run_id): services + for xp_id, services in SERVICES.items() + } | FORMER_RUN_SERVICES + for (xp_id, run_id), services in runs_services.items(): + self.services.setdefault(xp_id, []).extend( MockService( service_id, description, @@ -317,7 +325,7 @@ def __init__(self, workspace: Path): run_id=run_id, ) for service_id, description in services - ] + ) def _add_job(self, spec: JobSpec) -> MockJob: path = self.workspace / "jobs" / spec.task_id / spec.identifier @@ -502,7 +510,7 @@ def _fetch_orphan_jobs(self, *, refresh=False, **kwargs): def _fetch_services_from_storage(self, experiment_id, run_id): if experiment_id is None: return [s for services in self.services.values() for s in services] - return list(self.services.get(experiment_id, [])) + return [s for s in self.services.get(experiment_id, []) if s.run_id == run_id] # --- Operations diff --git a/src/experimaestro/tests/ui/test_services.py b/src/experimaestro/tests/ui/test_services.py index 94a588b3..ea121a50 100644 --- a/src/experimaestro/tests/ui/test_services.py +++ b/src/experimaestro/tests/ui/test_services.py @@ -7,7 +7,7 @@ import pytest -from .drivers import refused +from .drivers import Notification, refused pytestmark = pytest.mark.anyio @@ -55,3 +55,22 @@ async def test_active_services_shown_everywhere(ui): await ui.stop_service("sweep", "tensorboard") assert await ui.active_services() == {} + + +@pytest.mark.ui("web") +async def test_services_of_a_former_run(ui): + """The services of a former run are listed and started with its run id""" + former = "20260101_100000" + assert await ui.services("baseline", former) == {"notebook": "stopped"} + assert await ui.services("baseline") == {"tensorboard": "stopped"} + + # Not a service of the current run + assert await ui.start_service("baseline", "notebook") == Notification( + "error", "Service notebook not found" + ) + started = await ui.start_service("baseline", "notebook", former) + assert started.severity == "info" + assert await ui.services("baseline", former) == {"notebook": "running"} + stopped = await ui.stop_service("baseline", "notebook", former) + assert stopped.severity == "info" + assert await ui.services("baseline", former) == {"notebook": "stopped"} diff --git a/src/experimaestro/tests/ui/web_driver.py b/src/experimaestro/tests/ui/web_driver.py index dac2a251..3db9ee21 100644 --- a/src/experimaestro/tests/ui/web_driver.py +++ b/src/experimaestro/tests/ui/web_driver.py @@ -420,9 +420,14 @@ async def graph( # --- Services - async def services(self, experiment_id: str) -> dict[str, str]: + async def services( + self, experiment_id: str, run_id: str | None = None + ) -> dict[str, str]: self.services_by_key.clear() - await self._request("services", experimentId=experiment_id) + payload = {"experimentId": experiment_id} + if run_id is not None: + payload["runId"] = run_id + await self._request("services", **payload) return { service["id"]: service["state"].lower() for service in self.services_by_key.values() @@ -432,15 +437,21 @@ async def services(self, experiment_id: str) -> dict[str, str]: async def active_services(self) -> dict[str, str]: raise NotImplementedError("the services strip is shown by the browser client") - async def start_service(self, experiment_id: str, service_id: str) -> Notification: - return await self._notification( - "service.start", experimentId=experiment_id, id=service_id - ) + async def start_service( + self, experiment_id: str, service_id: str, run_id: str | None = None + ) -> Notification: + payload = {"experimentId": experiment_id, "id": service_id} + if run_id is not None: + payload["runId"] = run_id + return await self._notification("service.start", **payload) - async def stop_service(self, experiment_id: str, service_id: str) -> Notification: - return await self._notification( - "service.stop", experimentId=experiment_id, id=service_id - ) + async def stop_service( + self, experiment_id: str, service_id: str, run_id: str | None = None + ) -> Notification: + payload = {"experimentId": experiment_id, "id": service_id} + if run_id is not None: + payload["runId"] = run_id + return await self._notification("service.stop", **payload) async def connection_state(self) -> str | None: # Receives what the server pushed diff --git a/src/experimaestro/webui/data/dag.js b/src/experimaestro/webui/data/dag.js index 1fed0193..65e055eb 100644 --- a/src/experimaestro/webui/data/dag.js +++ b/src/experimaestro/webui/data/dag.js @@ -1,2 +1,2 @@ /*! For license information please see dag.js.LICENSE.txt */ -!function(){"use strict";var e={2213:function(e,t,n){n(9479);var r=n(5602),o=n(7912);function a(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n