feat(browser): browser_sign_in, and browser steps reach a multi env's computers (FD-4944)

src/agent_env/browsers/README.md

@@ -26,6 +26,13 @@ values, the origin's other keys left as they are; `browser_clear_local_storage {
 `browser_open_tab {url, background?}`, in front, or with `background` behind the active tab, which stays active; and
 `browser_close_tab {url}` and `browser_activate_tab {url}`, on the one open tab at that URL as the browser shows it
 (scheme and host lowercase, no default port, `/` for an empty path), none or several failing the step with the tabs
-open listed. Preflight refuses a step on an env with no browser, none of the name given, or several and none named.
+open listed. `browser_sign_in {site, user}` signs the browser in to `site`, a website env deployed in the run on its
+own or in a multi env, by its env id: the site's `website/v1` `session {user}` gives the cookies and local storage,
+each cookie is set for every name the site is reached by (an exact name by its URL, a `*.` name as a domain cookie),
+and the browser's `data/add` loads them, which opens the start tabs again while no agent has acted. A task runs it
+after the load that seeds the site and again after `reset_env`, which starts the browser afresh. Preflight refuses a
+step on an env with no browser, none of the name given, or several and none named. On a multi env, the steps reach
+the browser apps of its computers, each through the computer's own record and control token; `browser_connect_cdp`
+doesn't reach those yet.
 `browser_connect_cdp` hands an agent a CDP address to the same browser. A whole state, Playwright's `storageState`
 included, loads through `load_artifact` or `data/add` instead.

src/agent_env/env/envs/computer/utils/sites.py

@@ -509,7 +509,7 @@ async def deployed_sites(deployed_envs: list[DeployedEnv], only: Optional[list[s
         for child in children:
             params = protocol_v1.extension_params(child.card, WEBSITE_EXTENSION_URI)
             browse = params.get(WEBSITE_BROWSE_URL_PARAM)
-            service = _served_as(child)
+            service = served_as(child)
             if services is not None and service not in services:
                 named.add(service)
                 continue
@@ -589,7 +589,7 @@ def _env_qualified(found: list[tuple[str, dict]]) -> list[dict]:
             if len(envs[site["name"]]) > 1 else site for env_id, site in found]
 
 
-def _served_as(child) -> str:
+def served_as(child) -> str:
     """The name a website child env is served as, which ``website_frontend_urls`` keys it by: the environment_name its
     gateway states; from a gateway that states none, the ``<name>`` of the ``/svc/<name>/`` route its url is at, else
     its card's name."""

src/agent_env/task/episode/rules.py

@@ -150,7 +150,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s
 UNAFFECTED = frozenset({
     # They name no agent.
     "aggregate_verifiers", "apply_server_config", "browser_activate_tab", "browser_clear_local_storage",
-    "browser_close_tab", "browser_delete_cookies", "browser_open_tab", "browser_set_cookies",
+    "browser_close_tab", "browser_delete_cookies", "browser_open_tab", "browser_set_cookies", "browser_sign_in",
     "browser_set_local_storage", "build_mcp_cli", "combine_universe_verdicts", "configure_scheduled_checks",
     "deploy_env", "deploy_sandbox", "end_computer_recording", "start_computer_recording", "env_outcome_verifier",
     "modify_env_tool_access", "reset_env", "review",

src/agent_env/task_step/registry.py

@@ -29,6 +29,7 @@ def get_task_step_registry() -> dict[str, type["TaskStep"]]:
 def _builtin_registry() -> dict[str, type["TaskStep"]]:
     from agent_env.task_step.task_steps.browser.connect_cdp import BrowserConnectCdpTaskStep
     from agent_env.task_step.task_steps.browser.cookies import BrowserDeleteCookiesTaskStep, BrowserSetCookiesTaskStep
+    from agent_env.task_step.task_steps.browser.sign_in import BrowserSignInTaskStep
     from agent_env.task_step.task_steps.browser.local_storage import (
         BrowserClearLocalStorageTaskStep,
         BrowserSetLocalStorageTaskStep,
@@ -100,6 +101,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]:
         _get_type(BuildMcpCliTaskStep): BuildMcpCliTaskStep,
         _get_type(BrowserConnectCdpTaskStep): BrowserConnectCdpTaskStep,
         _get_type(BrowserSetCookiesTaskStep): BrowserSetCookiesTaskStep,
+        _get_type(BrowserSignInTaskStep): BrowserSignInTaskStep,
         _get_type(BrowserDeleteCookiesTaskStep): BrowserDeleteCookiesTaskStep,
         _get_type(BrowserSetLocalStorageTaskStep): BrowserSetLocalStorageTaskStep,
         _get_type(BrowserClearLocalStorageTaskStep): BrowserClearLocalStorageTaskStep,

src/agent_env/task_step/task_steps/browser/base.py

@@ -1,5 +1,6 @@
-"""BrowserStep: a task step on the website browser in a deployed env, found among the env's children by its card, and
-called through browser/v1 with the env's control token."""
+"""BrowserStep: a task step on the website browser in a deployed env, found among the env's children by its card, or
+among those of a multi env's computers by each computer's own record, and called through browser/v1 with the control
+token of the record that serves it."""
 
 from __future__ import annotations
 
@@ -10,6 +11,7 @@
 
 from agent_env.env.env import DeployedEnv, Env
 from agent_env.env.envs.computer.env import ComputerEnv
+from agent_env.env.envs.multi_env import MultiEnv
 from agent_env.task_step.context import TaskStepContext
 from agent_env.task_step.preflight import stored_env
 from agent_env.task_step.task_step import TaskStep, TaskStepDependency
@@ -65,9 +67,9 @@ def _browser_from_dict(cls, data: dict) -> dict:
         return {**cls._base_from_dict(data), "env_id": data["env_id"], "browser": data.get("browser")}
 
     def _find(self, context: TaskStepContext) -> tuple[DeployedEnv, Optional[str], dict]:
-        """The deployed env, and its browser's name, None for the env itself, and card."""
-        deployed = deployed_env(context, self.env_id)
-        return (deployed, *find_browser(deployed, self.browser))
+        """The record that serves the browser, the deployed env's or a multi env's computer's, and the browser's name,
+        None for that record's env itself, and card."""
+        return find_browser(deployed_env(context, self.env_id), self.browser)
 
     async def _call(self, context: TaskStepContext, op: str, params: dict,
                     timeout_s: float = CALL_TIMEOUT_S) -> Any:
@@ -86,7 +88,9 @@ def _record(self, context: TaskStepContext, **fields: Any) -> None:
 
 def stored_browsers(env: Env) -> list[str]:
     """The names of the website browsers a stored env deploys: a computer env's browser apps, the app itself for one
-    deployed alone; none for any other env."""
+    deployed alone, a multi env's computers' browser apps; none for any other env."""
+    if isinstance(env, MultiEnv):
+        return [name for computer in env.computer_envs for name in stored_browsers(computer)]
     if not isinstance(env, ComputerEnv):
         return []
     return [app.environment_name for app in env.composed_apps() if app.declaration.browser]
@@ -99,19 +103,28 @@ def deployed_env(context: TaskStepContext, env_id: str) -> DeployedEnv:
     return deployed
 
 
-def find_browser(deployed: DeployedEnv, named: Optional[str]) -> tuple[Optional[str], dict]:
-    """The website browser in ``deployed``, the one ``named`` else its only one: its name among the env's children,
-    or None for the env itself, and its card."""
-    card = deployed.environment_card or {}
-    browsers = [(None, card)] if find_extension(card, BROWSER) else []
-    browsers += [(child.get("name"), child) for child in card.get("children_environments") or []
-                 if find_extension(child, BROWSER)]
+def find_browser(deployed: DeployedEnv, named: Optional[str]) -> tuple[DeployedEnv, Optional[str], dict]:
+    """The website browser in ``deployed``, the one ``named`` else its only one: the record that serves it, its name
+    among that record's children, or None for the record's env itself, and its card. A multi env's gateway lists its
+    computers but not their apps, so a browser in one is found by the computer's own record, kept in the multi env's
+    (``DeployedGatewayEnv.child_envs``), whose control token reaches it."""
+    browsers = _browsers(deployed)
+    for data in (getattr(deployed, "child_envs", None) or {}).values():
+        browsers += _browsers(DeployedEnv.from_dict(data))
     if named is not None:
-        found = next((browser for browser in browsers if browser[0] == named), None)
+        found = next((browser for browser in browsers if browser[1] == named), None)
         if found is None:
             raise RuntimeError(f"env {deployed.env_id!r} has no website browser named {named!r}")
         return found
     if len(browsers) != 1:
         raise RuntimeError(f"env {deployed.env_id!r} has {len(browsers) or 'no'} website browsers"
                            + (": name one as browser" if browsers else ""))
     return browsers[0]
+
+
+def _browsers(record: DeployedEnv) -> list[tuple[DeployedEnv, Optional[str], dict]]:
+    """The website browsers ``record``'s card offers: its env itself, and each of its children that is one."""
+    card = record.environment_card or {}
+    found = [(record, None, card)] if find_extension(card, BROWSER) else []
+    return found + [(record, child.get("name"), child) for child in card.get("children_environments") or []
+                    if find_extension(child, BROWSER)]

src/agent_env/task_step/task_steps/browser/connect_cdp.py

@@ -75,6 +75,10 @@ def from_dict(cls, data: dict) -> BrowserConnectCdpTaskStep:
 
     async def execute(self, context: TaskStepContext) -> TaskStepContext:
         deployed, browser, _ = self._find(context)
+        if deployed.env_id != self.env_id:
+            raise RuntimeError(f"the website browser {browser!r} is in a computer of the multi env {self.env_id!r}, whose "
+                               "agents reach it through the multi env's gateway; browser_connect_cdp doesn't hand out "
+                               "that address yet")
         agent = _deployed_agent(context, self.agent_name)
         cdp_config = A2AAgent.find_extension(agent.a2a_card or {}, A2AAgent.EXT_CDP_CONFIG)
         if cdp_config is None or not agent.a2a_url:

src/agent_env/task_step/task_steps/browser/sign_in.py

@@ -0,0 +1,155 @@
+"""browser_sign_in: an env's website browser signed in to a site as a user, with the cookies and local storage the
+site's website/v1 session gives, scoped to the site's names and loaded through the browser's data/add."""
+
+from __future__ import annotations
+
+import logging
+from typing import ClassVar, Optional
+
+from agentenv_protocol import METHOD_ADD, DataPart, client as protocol_v1
+from agentenv_protocol.website import WEBSITE_EXTENSION_URI
+
+from agent_env.entity.refs import EntityRef
+from agent_env.env.env import DeployedEnv, Env, website_child
+from agent_env.env.envs.computer.utils import sites
+from agent_env.env.envs.website import WebsiteEnv
+from agent_env.store.base import NotFoundError
+from agent_env.task_step.context import TaskStepContext
+from agent_env.task_step.preflight import stored_env
+from agent_env.task_step.task_step import TaskStepDependency
+from agent_env.task_step.task_steps.browser.base import BrowserStep
+
+logger = logging.getLogger(__name__)
+
+#: How long a site may take to answer for a session, and the browser to load it.
+SESSION_TIMEOUT_S = 30
+ADD_TIMEOUT_S = 120
+
+
+class BrowserSignInTaskStep(BrowserStep):
+    """Sign the browser in to the website env ``site``, deployed in the run on its own or in a multi env, as ``user``:
+    the site's ``session {user}`` gives the cookies and local storage, each cookie is set for every name the site is
+    reached by, and the browser's ``data/add`` loads them, which opens its start tabs again while no agent has acted.
+    A task runs it after the load that seeds the site, and again after a reset, which starts the browser afresh."""
+
+    type: ClassVar[str] = "browser_sign_in"
+    entity_refs = (EntityRef.env("env_id"), EntityRef.env("site"))
+
+    def __init__(self, id: str, version: Optional[int], env_id: str, site: str, user: str,
+                 browser: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None,
+                 fail_task_on_error: bool = True):
+        super().__init__(id, version, env_id, browser, depends_on=depends_on, fail_task_on_error=fail_task_on_error)
+        self.site = site
+        self.user = user
+
+    def config_problems(self) -> list[str]:
+        problems = super().config_problems()
+        if not isinstance(self.site, str) or not self.site:
+            problems.append(f"step {self.id!r}: site must be a website env's id")
+        if not isinstance(self.user, str) or not self.user:
+            problems.append(f"step {self.id!r}: user must be a non-empty string; browser_delete_cookies signs out")
+        return problems
+
+    def preflight(self) -> list[str]:
+        problems = super().preflight()
+        if not isinstance(self.site, str) or not self.site:
+            return problems
+        site = stored_env(self.site)
+        if site is not None and not isinstance(site, WebsiteEnv):
+            problems.append(f"step {self.id!r}: site {self.site!r} is a {type(site).__name__}, not a website env")
+        return problems
+
+    def to_dict(self) -> dict:
+        return {**super().to_dict(), "site": self.site, "user": self.user}
+
+    @classmethod
+    def from_dict(cls, data: dict) -> BrowserSignInTaskStep:
+        return cls(**cls._browser_from_dict(data), site=data.get("site"), user=data.get("user"))
+
+    async def execute(self, context: TaskStepContext) -> TaskStepContext:
+        record, child, names = await self._site(context)
+        if protocol_v1.find_extension_method(child.card, WEBSITE_EXTENSION_URI, "session") is None:
+            raise RuntimeError(f"site {self.site!r} serves no website/v1 session, so it can't sign a browser in")
+        session = await record.invoke(WEBSITE_EXTENSION_URI, "session", {"user": self.user}, child=child,
+                                      timeout=SESSION_TIMEOUT_S)
+        cookies = scoped(session.get("cookies") or [], names)
+        owner, browser, card = self._find(context)
+        if protocol_v1.serves(card, METHOD_ADD) is False:
+            raise RuntimeError(f"the website browser {browser or owner.env_id!r} serves no data/add: rebuild it on a "
+                               "newer agentenv-framework-protocol")
+        state = {"cookies": cookies, "local_storage": session.get("local_storage") or []}
+        await protocol_v1.add_data(_data_plane(owner, browser), [DataPart(data=state)], timeout=ADD_TIMEOUT_S,
+                                   headers=owner.control_headers)
+        self._record(context, site=self.site, user=self.user,
+                     cookies=sorted({cookie["name"] for cookie in cookies}), names=names)
+        logger.info(f"{self.id}: signed {browser or owner.env_id!r} in to {self.site!r} as {self.user!r} "
+                    f"({len(cookies)} cookie(s) on {', '.join(names)})")
+        return context
+
+    async def _site(self, context: TaskStepContext) -> tuple[DeployedEnv, protocol_v1.ChildEnv, list[str]]:
+        """The record that serves the site, the site among its children, and the names a browser reaches it by: a
+        website env deployed on its own, or one of a deployed multi env's websites."""
+        for record in context.deployed_envs:
+            try:
+                env = Env.get(record.env_id, record.env_version)
+            except NotFoundError:
+                continue
+            website = env if record.env_id == self.site else next(
+                (site for site in getattr(env, "website_envs", None) or [] if site.id == self.site), None)
+            if not isinstance(website, WebsiteEnv):
+                continue
+            child = _served(record, website.environment_name)
+            if child is None:
+                raise RuntimeError(f"env {record.env_id!r}'s card serves no website as {website.environment_name!r}")
+            found = await sites.deployed_sites([record], services=[website.environment_name])
+            names = list(dict.fromkeys(name for site in found for name in site["domains"])) or _own_names(website, child)
+            if not names:
+                raise RuntimeError(f"site {self.site!r} has no names a browser reaches it by")
+            return record, child, names
+        raise RuntimeError(f"site {self.site!r} isn't deployed in this run: no env in context.deployed_envs is it or "
+                           "has it among its websites")
+
+
+def _served(record: DeployedEnv, service: str) -> Optional[protocol_v1.ChildEnv]:
+    """The website ``record``'s card serves as ``service``, the name of the route it's at, which its card's name
+    needn't be (``sites.served_as``); a leaf card, a website's deployed alone, is its own."""
+    card = record.environment_card or {}
+    if not isinstance(card.get("children_environments"), list):
+        return protocol_v1.ChildEnv(name=card.get("name"), url=card.get("url"), card=card) if card else None
+    return next((child for child in protocol_v1.children(card)
+                 if website_child(child) and sites.served_as(child) == service), None)
+
+
+def _own_names(website: WebsiteEnv, child: protocol_v1.ChildEnv) -> list[str]:
+    """The names ``website`` stores, else those its card's ``website/v1`` declares, for a site ``deployed_sites``
+    doesn't route: one a plugin deploys alone, with a card of its own and no frontend."""
+    declared = protocol_v1.extension_params(child.card, WEBSITE_EXTENSION_URI).get("domains")
+    names = website.domains or (declared if isinstance(declared, list) else [])
+    return list(dict.fromkeys(name for name in names if isinstance(name, str) and name))
+
+
+def scoped(cookies: list[dict], names: list[str]) -> list[dict]:
+    """Each of ``cookies``, which ``session`` leaves unplaced, set for each of the site's ``names``: an exact name by
+    its URL, so the cookie is the name's alone, over HTTPS, a ``.localhost`` one over HTTP; a ``*.`` name as a domain
+    cookie on the name it ends in."""
+    placed = []
+    for cookie in cookies:
+        unplaced = {key: value for key, value in cookie.items() if key not in ("url", "domain")}
+        for name in names:
+            if name.startswith("*."):
+                placed.append({**unplaced, "domain": f".{name[2:]}", "path": unplaced.get("path") or "/"})
+            else:
+                scheme = "http" if name == "localhost" or name.endswith(".localhost") else "https"
+                placed.append({**{key: value for key, value in unplaced.items() if key != "path"},
+                               "url": f"{scheme}://{name}{unplaced.get('path') or '/'}"})
+    return placed
+
+
+def _data_plane(owner: DeployedEnv, browser: Optional[str]) -> str:
+    """Where the browser's data plane is: its child's in ``owner``'s card, or ``owner``'s own for a browser alone."""
+    if browser is None:
+        return owner.environment_url
+    child = owner.child_env(browser)
+    if child is None:
+        raise RuntimeError(f"env {owner.env_id!r}'s card lists no {browser!r}")
+    return child.base_url(owner.environment_url)

tst/data/session_website/Dockerfile

@@ -0,0 +1,12 @@
+FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf
+RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+RUN pip install --no-cache-dir starlette uvicorn pydantic httpx
+COPY agentenv_protocol /app/agentenv_protocol
+COPY server.py /app/server.py
+ENV PYTHONPATH=/app
+ENV PORT=8000
+EXPOSE 8000
+RUN useradd --create-home --uid 10001 appuser
+USER appuser
+CMD ["python", "server.py"]

tst/data/session_website/server.py

@@ -0,0 +1,12 @@
+"""A website on WebsiteEnvironment with no pages of its own, named shop.test, for integration-testing browser_sign_in:
+website/v1's session signs a browser in, and /api/session says who a request's cookie signs in."""
+from agentenv_protocol.website import WebsiteEnvironment, website
+
+
+@website(domains=["shop.test"])
+class Shop(WebsiteEnvironment):
+    pass
+
+
+if __name__ == "__main__":
+    Shop().serve()