Improved display of prerequisite errors when running eval() from a …

…script or notebook (UKGovernmentBEIS#349)

CHANGELOG.md

@@ -6,6 +6,7 @@
 - [score()](https://inspect.ai-safety-institute.org.uk/solvers.html#sec-scoring-in-solvers) function for accessing scoring logic from within solvers.
 - `system_message()` now supports custom parameters and interpolation of `metadata` values from `Sample`.
 - Properly support metrics that return a dict or list of values
+- Improved display of prerequisite errors when running `eval()` from a script or notebook.
 - Added [SQuAD](https://github.com/UKGovernmentBEIS/inspect_ai/tree/main/benchmarks/squad) and [AGIEval](https://github.com/UKGovernmentBEIS/inspect_ai/tree/main/benchmarks/agieval) benchmarks.
 
 

src/inspect_ai/_cli/main.py

@@ -1,9 +1,7 @@
-import sys
-
 import click
 
 from inspect_ai._util.dotenv import init_dotenv
-from inspect_ai._util.error import exception_hook
+from inspect_ai._util.error import set_exception_hook
 
 from .. import __version__
 from .cache import cache_command
@@ -49,7 +47,7 @@ def inspect(ctx: click.Context, version: bool) -> None:
 
 
 def main() -> None:
-    sys.excepthook = exception_hook()
+    set_exception_hook()
     init_dotenv()
     inspect(auto_envvar_prefix="INSPECT")
 

src/inspect_ai/_util/error.py

@@ -67,3 +67,13 @@ def handler(
             sys_handler(exception_type, exception, traceback)
 
     return handler
+
+
+_exception_hook_set: bool = False
+
+
+def set_exception_hook() -> None:
+    global _exception_hook_set
+    if not _exception_hook_set:
+        sys.excepthook = exception_hook()
+        _exception_hook_set = True

src/inspect_ai/_util/platform.py

@@ -1,6 +1,8 @@
 import importlib.util
 import os
 
+from .error import set_exception_hook
+
 
 def running_in_notebook() -> bool:
     try:
@@ -16,6 +18,9 @@ def running_in_notebook() -> bool:
 
 
 def platform_init() -> None:
+    # set exception hook if we haven't already
+    set_exception_hook()
+
     # if we are running in a notebook, confirm that we have ipywidgets
     if running_in_notebook():
         # check for required packages