pybind11/tests/test_multiple_interpreters.py

from __future__ import annotations

import contextlib
import os
import pickle
import sys
import textwrap

import pytest

import env
import pybind11_tests

if env.IOS:
    pytest.skip("Subinterpreters not supported on iOS", allow_module_level=True)

# 3.14.0b3+, though sys.implementation.supports_isolated_interpreters is being added in b4
# Can be simplified when we drop support for the first three betas
CONCURRENT_INTERPRETERS_SUPPORT = (
    sys.version_info >= (3, 14)
    and (
        sys.version_info != (3, 14, 0, "beta", 1)
        and sys.version_info != (3, 14, 0, "beta", 2)
    )
    and (
        sys.version_info == (3, 14, 0, "beta", 3)
        or sys.implementation.supports_isolated_interpreters
    )
)


def get_interpreters(*, modern: bool):
    if modern and CONCURRENT_INTERPRETERS_SUPPORT:
        from concurrent import interpreters

        def create():
            return contextlib.closing(interpreters.create())

        def run_string(
            interp: interpreters.Interpreter,
            code: str,
            *,
            shared: dict[str, object] | None = None,
        ) -> Exception | None:
            if shared:
                interp.prepare_main(**shared)
            try:
                interp.exec(code)
                return None
            except interpreters.ExecutionFailed as err:
                return err

        return run_string, create

    if sys.version_info >= (3, 12):
        interpreters = pytest.importorskip(
            "_interpreters" if sys.version_info >= (3, 13) else "_xxsubinterpreters"
        )

        @contextlib.contextmanager
        def create(config: str = ""):
            try:
                if config:
                    interp = interpreters.create(config)
                else:
                    interp = interpreters.create()
            except TypeError:
                pytest.skip(f"interpreters module needs to support {config} config")

            try:
                yield interp
            finally:
                interpreters.destroy(interp)

        def run_string(
            interp: int, code: str, shared: dict[str, object] | None = None
        ) -> Exception | None:
            kwargs = {"shared": shared} if shared else {}
            return interpreters.run_string(interp, code, **kwargs)

        return run_string, create

    pytest.skip("Test requires the interpreters stdlib module")


@pytest.mark.skipif(
    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
)
def test_independent_subinterpreters():
    """Makes sure the internals object differs across independent subinterpreters"""

    sys.path.insert(0, os.path.dirname(pybind11_tests.__file__))

    run_string, create = get_interpreters(modern=True)

    import mod_per_interpreter_gil as m

    if not m.defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT:
        pytest.skip("Does not have subinterpreter support compiled in")

    code = textwrap.dedent(
        """
        import mod_per_interpreter_gil as m
        import pickle
        with open(pipeo, 'wb') as f:
            pickle.dump(m.internals_at(), f)
        """
    ).strip()

    with create() as interp1, create() as interp2:
        try:
            res0 = run_string(interp1, "import mod_shared_interpreter_gil")
            if res0 is not None:
                res0 = str(res0)
        except Exception as e:
            res0 = str(e)

        pipei, pipeo = os.pipe()
        run_string(interp1, code, shared={"pipeo": pipeo})
        with open(pipei, "rb") as f:
            res1 = pickle.load(f)

        pipei, pipeo = os.pipe()
        run_string(interp2, code, shared={"pipeo": pipeo})
        with open(pipei, "rb") as f:
            res2 = pickle.load(f)

    assert "does not support loading in subinterpreters" in res0, (
        "cannot use shared_gil in a default subinterpreter"
    )
    assert res1 != m.internals_at(), "internals should differ from main interpreter"
    assert res2 != m.internals_at(), "internals should differ from main interpreter"
    assert res1 != res2, "internals should differ between interpreters"


@pytest.mark.skipif(
    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
)
@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
def test_independent_subinterpreters_modern():
    """Makes sure the internals object differs across independent subinterpreters. Modern (3.14+) syntax."""

    sys.path.insert(0, os.path.dirname(pybind11_tests.__file__))

    import mod_per_interpreter_gil as m

    if not m.defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT:
        pytest.skip("Does not have subinterpreter support compiled in")

    from concurrent import interpreters

    code = textwrap.dedent(
        """
        import mod_per_interpreter_gil as m

        values.put_nowait(m.internals_at())
        """
    ).strip()

    with (
        contextlib.closing(interpreters.create()) as interp1,
        contextlib.closing(interpreters.create()) as interp2,
    ):
        with pytest.raises(
            interpreters.ExecutionFailed,
            match="does not support loading in subinterpreters",
        ):
            interp1.exec("import mod_shared_interpreter_gil")

        values = interpreters.create_queue()
        interp1.prepare_main(values=values)
        interp1.exec(code)
        res1 = values.get_nowait()

        interp2.prepare_main(values=values)
        interp2.exec(code)
        res2 = values.get_nowait()

    assert res1 != m.internals_at(), "internals should differ from main interpreter"
    assert res2 != m.internals_at(), "internals should differ from main interpreter"
    assert res1 != res2, "internals should differ between interpreters"


@pytest.mark.skipif(
    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
)
def test_dependent_subinterpreters():
    """Makes sure the internals object differs across subinterpreters"""

    sys.path.insert(0, os.path.dirname(pybind11_tests.__file__))

    run_string, create = get_interpreters(modern=False)

    import mod_shared_interpreter_gil as m

    if not m.defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT:
        pytest.skip("Does not have subinterpreter support compiled in")

    code = textwrap.dedent(
        """
        import mod_shared_interpreter_gil as m
        import pickle
        with open(pipeo, 'wb') as f:
            pickle.dump(m.internals_at(), f)
        """
    ).strip()

    with create("legacy") as interp1:
        pipei, pipeo = os.pipe()
        run_string(interp1, code, shared={"pipeo": pipeo})
        with open(pipei, "rb") as f:
            res1 = pickle.load(f)

    assert res1 != m.internals_at(), "internals should differ from main interpreter"


PREAMBLE_CODE = textwrap.dedent(
    f"""
    def test():
        import sys

        sys.path.insert(0, {os.path.dirname(env.__file__)!r})
        sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r})

        import collections
        import mod_per_interpreter_gil_with_singleton as m

        objects = m.get_objects_in_singleton()
        expected = [
            type(None),               # static type: shared between interpreters
            tuple,                    # static type: shared between interpreters
            list,                     # static type: shared between interpreters
            dict,                     # static type: shared between interpreters
            collections.OrderedDict,  # static type: shared between interpreters
            collections.defaultdict,  # heap type: dynamically created per interpreter
            collections.deque,        # heap type: dynamically created per interpreter
        ]
        # Check that we have the expected objects. Avoid IndexError by checking lengths first.
        assert len(objects) == len(expected), (
            f"Expected {{expected!r}} ({{len(expected)}}), got {{objects!r}} ({{len(objects)}})."
        )
        # The first ones are static types shared between interpreters.
        assert objects[:-2] == expected[:-2], (
            f"Expected static objects {{expected[:-2]!r}}, got {{objects[:-2]!r}}."
        )
        # The last two are heap types created per-interpreter.
        # The expected objects are dynamically imported from `collections`.
        assert objects[-2:] == expected[-2:], (
            f"Expected heap objects {{expected[-2:]!r}}, got {{objects[-2:]!r}}."
        )

        assert hasattr(m, 'MyClass'), "Module missing MyClass"
        assert hasattr(m, 'MyGlobalError'), "Module missing MyGlobalError"
        assert hasattr(m, 'MyLocalError'), "Module missing MyLocalError"
        assert hasattr(m, 'MyEnum'), "Module missing MyEnum"
    """
).lstrip()


@pytest.mark.skipif(
    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
)
@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
def test_import_module_with_singleton_per_interpreter():
    """Tests that a singleton storing Python objects works correctly per-interpreter"""
    from concurrent import interpreters

    code = f"{PREAMBLE_CODE.strip()}\n\ntest()\n"
    with contextlib.closing(interpreters.create()) as interp:
        interp.exec(code)


@pytest.mark.skipif(
    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
)
@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
def test_import_in_subinterpreter_after_main():
    """Tests that importing a module in a subinterpreter after the main interpreter works correctly"""
    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
            import contextlib
            import gc
            from concurrent import interpreters

            test()

            interp = None
            with contextlib.closing(interpreters.create()) as interp:
                interp.call(test)

            del interp
            for _ in range(5):
                gc.collect()
            """
        )
    )

    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
            import contextlib
            import gc
            import random
            from concurrent import interpreters

            test()

            interps = interp = None
            with contextlib.ExitStack() as stack:
                interps = [
                    stack.enter_context(contextlib.closing(interpreters.create()))
                    for _ in range(8)
                ]
                random.shuffle(interps)
                for interp in interps:
                    interp.call(test)

            del interps, interp, stack
            for _ in range(5):
                gc.collect()
            """
        )
    )


@pytest.mark.skipif(
    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
)
@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
def test_import_in_subinterpreter_before_main():
    """Tests that importing a module in a subinterpreter before the main interpreter works correctly"""
    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
            import contextlib
            import gc
            from concurrent import interpreters

            interp = None
            with contextlib.closing(interpreters.create()) as interp:
                interp.call(test)

            test()

            del interp
            for _ in range(5):
                gc.collect()
            """
        )
    )

    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
            import contextlib
            import gc
            from concurrent import interpreters

            interps = interp = None
            with contextlib.ExitStack() as stack:
                interps = [
                    stack.enter_context(contextlib.closing(interpreters.create()))
                    for _ in range(8)
                ]
                for interp in interps:
                    interp.call(test)

            test()

            del interps, interp, stack
            for _ in range(5):
                gc.collect()
            """
        )
    )

    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
            import contextlib
            import gc
            from concurrent import interpreters

            interps = interp = None
            with contextlib.ExitStack() as stack:
                interps = [
                    stack.enter_context(contextlib.closing(interpreters.create()))
                    for _ in range(8)
                ]
                for interp in interps:
                    interp.call(test)

                test()

            del interps, interp, stack
            for _ in range(5):
                gc.collect()
            """
        )
    )


@pytest.mark.skipif(
    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
)
@pytest.mark.xfail(
    env.MUSLLINUX,
    reason="Flaky on musllinux, see also: https://github.com/pybind/pybind11/pull/5972#discussion_r2755283335",
    strict=False,
)
@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
def test_import_in_subinterpreter_concurrently():
    """Tests that importing a module in multiple subinterpreters concurrently works correctly"""
    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
            import gc
            from concurrent.futures import InterpreterPoolExecutor, as_completed

            futures = future = None
            with InterpreterPoolExecutor(max_workers=16) as executor:
                futures = [executor.submit(test) for _ in range(32)]
                for future in as_completed(futures):
                    future.result()
            del futures, future, executor

            for _ in range(5):
                gc.collect()
            """
        )
    )