fix(notebooks): make matplotlib list its backends again after Colab installs ipympl

docs/notebooks/_rtb_nb_bootstrap.py

@@ -76,6 +76,24 @@ async def ensure_installed(extras: tuple[str, ...] | list[str] = ()) -> bool:
                 [sys.executable, "-m", "pip", "install", "-q", requirement, "ipympl"],
                 check=True,
             )
+            # matplotlib renames the module://ipympl... backend to "widget", and looks
+            # it up in a list of backends that it builds once, from the installed
+            # packages, the first time it needs it.  Colab imported matplotlib before
+            # ipympl was installed, so %matplotlib ipympl is rejected ("... is not a
+            # valid value for backend") until the session restarts.  Make matplotlib
+            # build the list again.  Clearing the registry is what its own tests do,
+            # and it is needed: only resetting the flag registers the backends that
+            # were already there (matplotlib-inline's) twice, which is an error.
+            # _clear is private, hence the guards.
+            try:
+                from matplotlib.backends import backend_registry
+            except ImportError:
+                pass  # an older matplotlib has no registry to refresh
+            else:
+                clear = getattr(backend_registry, "_clear", None)
+                if clear is not None:
+                    clear()
+
             # ipympl figures need Colab's custom widget manager, enabled before the
             # first figure is created.
             from google.colab import output

docs/notebooks/branched-robot.ipynb

@@ -67,6 +67,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

docs/notebooks/dynamics.ipynb

@@ -66,6 +66,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

docs/notebooks/ets.ipynb

@@ -68,6 +68,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

docs/notebooks/graphics-test.ipynb

@@ -65,6 +65,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

docs/notebooks/icra2021.ipynb

@@ -69,6 +69,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

docs/notebooks/icra2026.ipynb

@@ -67,6 +67,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

docs/notebooks/ik-symbolic.ipynb

@@ -84,6 +84,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

docs/notebooks/iros2020.ipynb

@@ -75,6 +75,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

docs/notebooks/kinematics.ipynb

@@ -67,6 +67,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

docs/notebooks/symbolic.ipynb

@@ -67,6 +67,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

docs/notebooks/two-link-dynamics.ipynb

@@ -81,6 +81,14 @@
     "            if extras:\n",
     "                requirement += '[' + ','.join(extras) + ']'\n",
     "            subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', requirement, 'ipympl'], check=True)\n",
+    "            try:\n",
+    "                from matplotlib.backends import backend_registry\n",
+    "            except ImportError:\n",
+    "                pass\n",
+    "            else:\n",
+    "                clear = getattr(backend_registry, '_clear', None)\n",
+    "                if clear is not None:\n",
+    "                    clear()\n",
     "            from google.colab import output\n",
     "            output.enable_custom_widget_manager()\n",
     "            where, colab = ('on Colab', True)\n",

tests/test_notebook_bootstrap.py

@@ -356,11 +356,11 @@ def test_the_browser_install_goes_through_piplite():
     assert '"/pypi"' not in template
 
 
-def test_the_colab_install_includes_ipympl_and_the_widget_manager(monkeypatch):
-    # Colab does not ship ipympl, and matplotlib rejects "%matplotlib ipympl" without
-    # it.  The Colab branch is run here with a fake google.colab and no real pip, so
-    # this checks the command and the widget-manager call, not a real Colab install.
-    import asyncio
+def _colab_bootstrap(monkeypatch):
+    """The bootstrap module with a fake google.colab and no real pip.
+
+    :returns: the module, the pip commands it ran, and the widget-manager calls
+    """
     import types
 
     bootstrap = _load_module("_rtb_nb_bootstrap", NOTEBOOKS_DIR / "_rtb_nb_bootstrap.py")
@@ -385,10 +385,69 @@ def fake_run(cmd, **kw):
 
     monkeypatch.setattr(bootstrap.subprocess, "run", fake_run)
     monkeypatch.setattr(bootstrap.sys, "platform", "linux")
+    return bootstrap, calls, enabled
+
+
+def test_the_colab_install_includes_ipympl_and_the_widget_manager(monkeypatch):
+    # Colab does not ship ipympl, and matplotlib rejects "%matplotlib ipympl" without
+    # it.  The Colab branch is run here with a fake google.colab and no real pip, so
+    # this checks the command and the widget-manager call, not a real Colab install.
+    import asyncio
+
+    bootstrap, calls, enabled = _colab_bootstrap(monkeypatch)
 
     assert asyncio.run(bootstrap.ensure_installed(["ik"])) is True
 
     assert len(calls) == 1
     assert "roboticstoolbox-python[ik]" in calls[0]
     assert "ipympl" in calls[0]
     assert enabled == [True]
+
+
+def test_the_colab_install_makes_matplotlib_list_its_backends_again(monkeypatch):
+    # matplotlib builds its list of backends once.  Colab imports matplotlib before
+    # ipympl is installed, so the list lacks ipympl and %matplotlib ipympl is rejected
+    # until the session restarts.  This checks that the registry is cleared after the
+    # install; the real failure needs a fresh Colab runtime (see the PR for how it
+    # was reproduced in an isolated virtual environment).
+    import asyncio
+
+    from matplotlib.backends import backend_registry
+
+    bootstrap, _, _ = _colab_bootstrap(monkeypatch)
+    backend_registry.list_all()  # builds the list
+    assert backend_registry._loaded_entry_points is True
+
+    asyncio.run(bootstrap.ensure_installed())
+
+    assert backend_registry._loaded_entry_points is False
+    # and it can be built again, which resetting only the flag would not allow
+    backend_registry.list_all()
+
+
+def test_the_colab_install_copes_with_a_matplotlib_that_cannot_be_cleared(monkeypatch):
+    # _clear is private; if a matplotlib release drops it, the bootstrap must still run
+    import asyncio
+
+    from matplotlib.backends import backend_registry
+
+    bootstrap, _, enabled = _colab_bootstrap(monkeypatch)
+    monkeypatch.delattr(type(backend_registry), "_clear")
+
+    assert asyncio.run(bootstrap.ensure_installed()) is True
+    assert enabled == [True]
+
+
+def test_the_colab_install_copes_with_a_matplotlib_without_a_backend_registry(
+    monkeypatch,
+):
+    # matplotlib before 3.9 has no registry
+    import asyncio
+
+    import matplotlib.backends
+
+    bootstrap, _, enabled = _colab_bootstrap(monkeypatch)
+    monkeypatch.delattr(matplotlib.backends, "backend_registry")
+
+    assert asyncio.run(bootstrap.ensure_installed()) is True
+    assert enabled == [True]