From e8782a8fe9832b0eba24e5d1779ef51feb38284a Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Sun, 6 Sep 2026 06:22:47 +0700 Subject: [PATCH 1/2] Add R04, frozen modules The fourth runtime lesson. R03 ended on the fact that import os never opens os.py, because FrozenImporter gets asked before PathFinder, so this one goes after the modules that live inside the binary. Ten code cells, twenty one citations, six diagrams, four glossary terms and two Tier 1 recordings. The spine is the chicken and egg problem. The import system is written in Python, in Lib/importlib/_bootstrap.py, so it cannot be imported. CPython cuts the loop by compiling that file during its own build, marshalling the code object and writing the bytes into the binary as a C array. The proof that this works is in the bytecode: the module body of _frozen_importlib contains zero IMPORT_NAME opcodes, while _frozen_importlib_external, which is loaded second and by then has an import system, contains eight. init_importlib hands sys and _imp in as arguments, which is why _bootstrap.py never writes import sys. A stock 3.15 build freezes thirty three names in three arrays that the flag treats differently. Three are the import system and no setting can remove them, because look_up_frozen walks that array unconditionally and only the other two sit behind a check. Nineteen are what a bare startup needs. Eleven are hello world modules for the test suite. A frozen module still knows where it came from. FrozenImporter.find_spec works the source path out from sys._stdlib_dir and parks it on the spec as loader_state, and the loader copies it onto __file__, so the spec says frozen and __file__ says a real path and both are true. That is what makes a traceback through frozen code readable: linecache sees a filename starting with . Switching the flag off in process with _imp._override_frozen_modules_for_tests moves os from FrozenImporter to SourceFileLoader and leaves the same constants behind. What freezing actually buys is the finder search and the file read, not the unmarshal, which is identical on both paths, and two recordings put it at about a tenth of a startup on a release build and a tenth on a debug build, where it is off by default | M8 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r04-frozen-modules/r04.ipynb) | More are landing in order. [lessons/README.md](lessons/README.md) explains how one is put together and how to run them locally. diff --git a/citations.lock.json b/citations.lock.json index 857aaf4..9ca83bf 100644 --- a/citations.lock.json +++ b/citations.lock.json @@ -1210,6 +1210,26 @@ "first_line": "def _parse_exception_table(code):", "lines": 15 }, + "Lib/importlib/_bootstrap.py:1000-1011@v3.15.0rc1": { + "digest": "6ccd8a9547a7d8c2", + "first_line": "class FrozenImporter:", + "lines": 12 + }, + "Lib/importlib/_bootstrap.py:1106-1133@v3.15.0rc1": { + "digest": "21e719984e0eaeb5", + "first_line": "def find_spec(cls, fullname, path=None, target=None):", + "lines": 28 + }, + "Lib/importlib/_bootstrap.py:1148-1153@v3.15.0rc1": { + "digest": "63fc1959306d2042", + "first_line": "@staticmethod", + "lines": 6 + }, + "Lib/importlib/_bootstrap.py:1155-1165@v3.15.0rc1": { + "digest": "04de1062682f64e4", + "first_line": "@classmethod", + "lines": 11 + }, "Lib/importlib/_bootstrap.py:1198-1223@v3.15.0rc1": { "digest": "10232b40fdaf68e3", "first_line": "def _find_spec(name, path, target=None):", @@ -1230,6 +1250,16 @@ "first_line": "def __import__(name, globals=None, locals=None, fromlist=(), level=0):", "lines": 35 }, + "Lib/importlib/_bootstrap.py:1501-1539@v3.15.0rc1": { + "digest": "175a08818ad49605", + "first_line": "def _setup(sys_module, _imp_module):", + "lines": 39 + }, + "Lib/importlib/_bootstrap.py:1541-1546@v3.15.0rc1": { + "digest": "a8ccfaac3e1ef22d", + "first_line": "def _install(sys_module, _imp_module):", + "lines": 6 + }, "Lib/importlib/_bootstrap.py:226-240@v3.15.0rc1": { "digest": "54c579dcaffdc305", "first_line": "class _ModuleLock:", @@ -1290,11 +1320,21 @@ "first_line": "def getsourcelines(object):", "lines": 20 }, + "Lib/inspect.py:3386-3400@v3.15.0rc1": { + "digest": "1460bc9e4c86a95e", + "first_line": "reported_target = reported_module_name", + "lines": 15 + }, "Lib/inspect.py:886-897@v3.15.0rc1": { "digest": "108c22eee97b22a8", "first_line": "def getsourcefile(object):", "lines": 12 }, + "Lib/linecache.py:122-140@v3.15.0rc1": { + "digest": "80a479410cfbbe4e", + "first_line": "entry = cache.pop(filename, None)", + "lines": 19 + }, "Lib/opcode.py:16-23@v3.15.0rc1": { "digest": "e4e79270a74738f6", "first_line": "from _opcode_metadata import (_specializations, _specialized_opmap, opmap, # noqa: F401", @@ -4135,6 +4175,21 @@ "first_line": "static void", "lines": 22 }, + "Python/frozen.c:127-137@v3.15.0rc1": { + "digest": "1992fe1133dc64e4", + "first_line": "static const struct _module_alias aliases[] = {", + "lines": 11 + }, + "Python/frozen.c:140-144@v3.15.0rc1": { + "digest": "50e0058411e2f488", + "first_line": "", + "lines": 5 + }, + "Python/frozen.c:74-79@v3.15.0rc1": { + "digest": "6c90a431b887b976", + "first_line": "static const struct _frozen bootstrap_modules[] = {", + "lines": 6 + }, "Python/gc.c:1011-1039@v3.15.0rc1": { "digest": "63bcba6da6794191", "first_line": "/* Handle uncollectable garbage (cycles with tp_del slots, and stuff reachable", @@ -4390,6 +4445,31 @@ "first_line": "static PyObject *", "lines": 14 }, + "Python/import.c:3104-3129@v3.15.0rc1": { + "digest": "d8230b80e5b80f4e", + "first_line": "static const struct _frozen *", + "lines": 26 + }, + "Python/import.c:3130-3150@v3.15.0rc1": { + "digest": "1af64cf35596eb29", + "first_line": "// Frozen stdlib modules may be disabled.", + "lines": 21 + }, + "Python/import.c:3152-3159@v3.15.0rc1": { + "digest": "51c97e549dbb2884", + "first_line": "struct frozen_info {", + "lines": 8 + }, + "Python/import.c:3210-3229@v3.15.0rc1": { + "digest": "96bd53170c95c4ee", + "first_line": "static PyObject *", + "lines": 20 + }, + "Python/import.c:3389-3428@v3.15.0rc1": { + "digest": "cc9ed48c09b8fb3d", + "first_line": "static int", + "lines": 40 + }, "Python/import.c:4190-4227@v3.15.0rc1": { "digest": "40a1544a3fd82833", "first_line": "PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals,", @@ -4830,6 +4910,11 @@ "first_line": "int", "lines": 36 }, + "Tools/build/freeze_modules.py:37-76@v3.15.0rc1": { + "digest": "9d354f9c964e4b4f", + "first_line": "FROZEN = [", + "lines": 40 + }, "Tools/cases_generator/README.md:14-34@v3.15.0rc1": { "digest": "fb06409a976b7a51", "first_line": "- `tierN_generator.py`: a couple of driver scripts to read `Python/bytecodes.c` and", diff --git a/experiments/README.md b/experiments/README.md index 1f54f8c..5db8137 100644 --- a/experiments/README.md +++ b/experiments/README.md @@ -42,6 +42,8 @@ So those programs run somewhere else. They run in the images this project publis | [r02-what-a-second-interpreter-costs-without-the-lock](tier1/r02-what-a-second-interpreter-costs-without-the-lock.md) | R02 | freethreaded | Does an interpreter cost the same to make and to keep on a build with no lock? | | [r03-how-much-of-an-import-is-parallel](tier1/r03-how-much-of-an-import-is-parallel.md) | R03 | release | Does the import lock stop two threads importing at once, or does something else? | | [r03-how-much-of-an-import-is-parallel-without-the-lock](tier1/r03-how-much-of-an-import-is-parallel-without-the-lock.md) | R03 | freethreaded | With the GIL out of the way, do four imports on four threads finish in the time of one? | +| [r04-what-freezing-saves-at-startup](tier1/r04-what-freezing-saves-at-startup.md) | R04 | release | What does compiling the standard library into the binary actually save at startup? | +| [r04-what-freezing-saves-on-a-debug-build](tier1/r04-what-freezing-saves-on-a-debug-build.md) | R04 | debug | Does a debug build behave the same way, and does freezing still pay for itself there? | ## The commands diff --git a/experiments/tier1/r04-what-freezing-saves-at-startup.md b/experiments/tier1/r04-what-freezing-saves-at-startup.md new file mode 100644 index 0000000..cfffbcb --- /dev/null +++ b/experiments/tier1/r04-what-freezing-saves-at-startup.md @@ -0,0 +1,165 @@ +# A startup with the frozen standard library, and the same startup without it + +Generated by `just build-tier1`. Do not edit by hand, the change will be overwritten. + +What does compiling the standard library into the binary actually save at startup? + +- Lesson: R04 +- Build: release +- Image: ghcr.io/tamnd/cpython-internals/cpython:release@sha256:fb55d6afcf053c974de6447fafbd2be6af20cdb9f596e25a0445607b8af981e3 +- Interpreter: 3.15.0rc1 (37e98da:37e98da, Aug 29 2026, 09:24:54) [GCC 14.2.0] +- Recorded: 2026-09-06 + +Why this needs the release build: it wants a machine that is not busy with anything else, because half of what it reports is wall clock for a process that only lives for a few milliseconds. + +## The program + +```python +"""What freezing the standard library into the binary is worth at startup. + +Import is written in Python, so it cannot be imported. CPython gets around that by compiling a +handful of modules during its own build and writing the bytecode into the binary as C arrays. The +first three are the import system itself, and they are what lets the interpreter get going at all. + +Everything after those three is a speed decision rather than a correctness one, and it can be +switched off with -X frozen_modules=off, so the cost of it can be measured rather than guessed. +Which way the switch sits by default is a build choice, so this program never relies on the +default. It asks for on and off explicitly and reports what the default happens to be. + +Two numbers matter. The first is how many code objects a bare startup reads off disk, which -v +prints one line for, so it is a count rather than a timing and it is the same on any machine. The +second is wall clock, measured with the two cases alternating, because anything that runs one case +forty times and then the other forty times is measuring the state of the page cache instead. +""" + +import _imp +import statistics +import subprocess +import sys +import time + +ROUNDS = 40 +ON = ["-X", "frozen_modules=on"] +OFF = ["-X", "frozen_modules=off"] +ORIGIN = "import os; print(os.__spec__.origin)" +BOOTSTRAP = "import sys; print(sys.modules['_frozen_importlib'].__spec__.origin)" +COUNT = ( + "import sys\n" + "specs = [getattr(m, '__spec__', None) for m in sys.modules.values()]\n" + "print(len(sys.modules), sum(1 for s in specs if s is not None and s.origin == 'frozen'))\n" +) + + +def child(flags, args): + """Run this same interpreter again with those flags and hand back the finished process.""" + return subprocess.run( + [sys.executable, *flags, *args], capture_output=True, text=True, check=True + ) + + +def origin(flags): + """Ask a fresh interpreter where the os module it just imported came from.""" + return child(flags, ["-c", ORIGIN]).stdout.strip() + + +def loaded(flags): + """Ask a fresh interpreter how many modules it loaded and how many came out of the binary.""" + return [int(part) for part in child(flags, ["-c", COUNT]).stdout.split()] + + +def files_read(flags): + """Count the code objects a fresh interpreter reads off disk, which -v prints one per line.""" + printed = child(flags, ["-v", "-c", "pass"]).stderr + return sum(1 for line in printed.splitlines() if line.startswith("# code object from")) + + +def import_work(flags): + """Add up the self time -X importtime reports, in microseconds.""" + printed = child(["-X", "importtime", *flags], ["-c", "pass"]).stderr + total = 0 + for line in printed.splitlines(): + if line.startswith("import time:"): + first = line.removeprefix("import time:").split("|")[0].strip() + if first.isdigit(): + total += int(first) + return total + + +def startup_ms(flags): + """Wall clock milliseconds for one whole interpreter startup that does nothing at all.""" + started = time.perf_counter() + child(flags, ["-c", "pass"]) + return (time.perf_counter() - started) * 1000 + + +def alternating(measure, rounds): + """Measure both cases once each per round, so neither one gets the cold cache every time.""" + got = {"on": [], "off": []} + for _ in range(rounds): + got["on"].append(measure(ON)) + got["off"].append(measure(OFF)) + return got + + +_imp._override_frozen_modules_for_tests(1) +names = _imp._frozen_module_names() +frozen_os = _imp.get_frozen_object("os") +_imp._override_frozen_modules_for_tests(0) + +print("names compiled into this binary:", len(names)) +print("the three that cannot be turned off:", ", ".join(names[:3])) +print("filename on the frozen code object for os:", frozen_os.co_filename) +print("bytes of bytecode in it:", len(frozen_os.co_code)) +print("this build uses them unless told otherwise:", origin([]) == "frozen") +print("where os comes from with the flag on:", origin(ON)) +print("where os comes from with the flag off:", origin(OFF).rpartition("/")[2]) +print("_frozen_importlib with the flag off:", child(OFF, ["-c", BOOTSTRAP]).stdout.strip()) + +on_total, on_frozen = loaded(ON) +off_total, off_frozen = loaded(OFF) +print("modules a bare startup loads, flag on:", on_total) +print("modules a bare startup loads, flag off:", off_total) +print("of those, frozen with the flag on:", on_frozen) +print("of those, frozen with the flag off:", off_frozen) +print("code objects read off disk with the flag on:", files_read(ON)) +print("code objects read off disk with the flag off:", files_read(OFF)) + +work = alternating(import_work, 10) +print(f"~ import work reported by importtime, frozen on: {min(work['on'])} us") +print(f"~ import work reported by importtime, frozen off: {min(work['off'])} us") + +runs = alternating(startup_ms, ROUNDS) +fast_on, fast_off = min(runs["on"]), min(runs["off"]) +print(f"~ fastest startup with the flag on: {fast_on:.1f} ms") +print(f"~ fastest startup with the flag off: {fast_off:.1f} ms") +print(f"~ middle startup with the flag on: {statistics.median(runs['on']):.1f} ms") +print(f"~ middle startup with the flag off: {statistics.median(runs['off']):.1f} ms") +share = (1 - fast_on / fast_off) * 100 +print(f"~ share of a startup that freezing gives back: {share:.1f} percent") +``` + +## What it printed + +```text +names compiled into this binary: 33 +the three that cannot be turned off: _frozen_importlib, _frozen_importlib_external, zipimport +filename on the frozen code object for os: +bytes of bytecode in it: 3360 +this build uses them unless told otherwise: True +where os comes from with the flag on: frozen +where os comes from with the flag off: os.py +_frozen_importlib with the flag off: frozen +modules a bare startup loads, flag on: 33 +modules a bare startup loads, flag off: 33 +of those, frozen with the flag on: 17 +of those, frozen with the flag off: 3 +code objects read off disk with the flag on: 0 +code objects read off disk with the flag off: 13 +~ import work reported by importtime, frozen on: 12884 us +~ import work reported by importtime, frozen off: 19167 us +~ fastest startup with the flag on: 30.5 ms +~ fastest startup with the flag off: 35.6 ms +~ middle startup with the flag on: 33.9 ms +~ middle startup with the flag off: 39.5 ms +~ share of a startup that freezing gives back: 14.2 percent +``` diff --git a/experiments/tier1/r04-what-freezing-saves-on-a-debug-build.md b/experiments/tier1/r04-what-freezing-saves-on-a-debug-build.md new file mode 100644 index 0000000..89b8450 --- /dev/null +++ b/experiments/tier1/r04-what-freezing-saves-on-a-debug-build.md @@ -0,0 +1,165 @@ +# The same two startups on a build that does not use the frozen copies + +Generated by `just build-tier1`. Do not edit by hand, the change will be overwritten. + +Does a debug build behave the same way, and does freezing still pay for itself there? + +- Lesson: R04 +- Build: debug +- Image: ghcr.io/tamnd/cpython-internals/cpython:debug@sha256:7baea8f3dd4de2e4c3b020543729b147e636494ae9758dabffb4675793e37170 +- Interpreter: 3.15.0rc1 (37e98da:37e98da, Aug 29 2026, 09:24:21) [GCC 14.2.0] +- Recorded: 2026-09-06 + +Why this needs the debug build: it needs a build configured with --with-pydebug, which is the one build that leaves the frozen copies switched off by default, so the same program reports a different default. + +## The program + +```python +"""What freezing the standard library into the binary is worth at startup. + +Import is written in Python, so it cannot be imported. CPython gets around that by compiling a +handful of modules during its own build and writing the bytecode into the binary as C arrays. The +first three are the import system itself, and they are what lets the interpreter get going at all. + +Everything after those three is a speed decision rather than a correctness one, and it can be +switched off with -X frozen_modules=off, so the cost of it can be measured rather than guessed. +Which way the switch sits by default is a build choice, so this program never relies on the +default. It asks for on and off explicitly and reports what the default happens to be. + +Two numbers matter. The first is how many code objects a bare startup reads off disk, which -v +prints one line for, so it is a count rather than a timing and it is the same on any machine. The +second is wall clock, measured with the two cases alternating, because anything that runs one case +forty times and then the other forty times is measuring the state of the page cache instead. +""" + +import _imp +import statistics +import subprocess +import sys +import time + +ROUNDS = 40 +ON = ["-X", "frozen_modules=on"] +OFF = ["-X", "frozen_modules=off"] +ORIGIN = "import os; print(os.__spec__.origin)" +BOOTSTRAP = "import sys; print(sys.modules['_frozen_importlib'].__spec__.origin)" +COUNT = ( + "import sys\n" + "specs = [getattr(m, '__spec__', None) for m in sys.modules.values()]\n" + "print(len(sys.modules), sum(1 for s in specs if s is not None and s.origin == 'frozen'))\n" +) + + +def child(flags, args): + """Run this same interpreter again with those flags and hand back the finished process.""" + return subprocess.run( + [sys.executable, *flags, *args], capture_output=True, text=True, check=True + ) + + +def origin(flags): + """Ask a fresh interpreter where the os module it just imported came from.""" + return child(flags, ["-c", ORIGIN]).stdout.strip() + + +def loaded(flags): + """Ask a fresh interpreter how many modules it loaded and how many came out of the binary.""" + return [int(part) for part in child(flags, ["-c", COUNT]).stdout.split()] + + +def files_read(flags): + """Count the code objects a fresh interpreter reads off disk, which -v prints one per line.""" + printed = child(flags, ["-v", "-c", "pass"]).stderr + return sum(1 for line in printed.splitlines() if line.startswith("# code object from")) + + +def import_work(flags): + """Add up the self time -X importtime reports, in microseconds.""" + printed = child(["-X", "importtime", *flags], ["-c", "pass"]).stderr + total = 0 + for line in printed.splitlines(): + if line.startswith("import time:"): + first = line.removeprefix("import time:").split("|")[0].strip() + if first.isdigit(): + total += int(first) + return total + + +def startup_ms(flags): + """Wall clock milliseconds for one whole interpreter startup that does nothing at all.""" + started = time.perf_counter() + child(flags, ["-c", "pass"]) + return (time.perf_counter() - started) * 1000 + + +def alternating(measure, rounds): + """Measure both cases once each per round, so neither one gets the cold cache every time.""" + got = {"on": [], "off": []} + for _ in range(rounds): + got["on"].append(measure(ON)) + got["off"].append(measure(OFF)) + return got + + +_imp._override_frozen_modules_for_tests(1) +names = _imp._frozen_module_names() +frozen_os = _imp.get_frozen_object("os") +_imp._override_frozen_modules_for_tests(0) + +print("names compiled into this binary:", len(names)) +print("the three that cannot be turned off:", ", ".join(names[:3])) +print("filename on the frozen code object for os:", frozen_os.co_filename) +print("bytes of bytecode in it:", len(frozen_os.co_code)) +print("this build uses them unless told otherwise:", origin([]) == "frozen") +print("where os comes from with the flag on:", origin(ON)) +print("where os comes from with the flag off:", origin(OFF).rpartition("/")[2]) +print("_frozen_importlib with the flag off:", child(OFF, ["-c", BOOTSTRAP]).stdout.strip()) + +on_total, on_frozen = loaded(ON) +off_total, off_frozen = loaded(OFF) +print("modules a bare startup loads, flag on:", on_total) +print("modules a bare startup loads, flag off:", off_total) +print("of those, frozen with the flag on:", on_frozen) +print("of those, frozen with the flag off:", off_frozen) +print("code objects read off disk with the flag on:", files_read(ON)) +print("code objects read off disk with the flag off:", files_read(OFF)) + +work = alternating(import_work, 10) +print(f"~ import work reported by importtime, frozen on: {min(work['on'])} us") +print(f"~ import work reported by importtime, frozen off: {min(work['off'])} us") + +runs = alternating(startup_ms, ROUNDS) +fast_on, fast_off = min(runs["on"]), min(runs["off"]) +print(f"~ fastest startup with the flag on: {fast_on:.1f} ms") +print(f"~ fastest startup with the flag off: {fast_off:.1f} ms") +print(f"~ middle startup with the flag on: {statistics.median(runs['on']):.1f} ms") +print(f"~ middle startup with the flag off: {statistics.median(runs['off']):.1f} ms") +share = (1 - fast_on / fast_off) * 100 +print(f"~ share of a startup that freezing gives back: {share:.1f} percent") +``` + +## What it printed + +```text +names compiled into this binary: 33 +the three that cannot be turned off: _frozen_importlib, _frozen_importlib_external, zipimport +filename on the frozen code object for os: +bytes of bytecode in it: 3360 +this build uses them unless told otherwise: False +where os comes from with the flag on: frozen +where os comes from with the flag off: os.py +_frozen_importlib with the flag off: frozen +modules a bare startup loads, flag on: 33 +modules a bare startup loads, flag off: 33 +of those, frozen with the flag on: 17 +of those, frozen with the flag off: 3 +code objects read off disk with the flag on: 0 +code objects read off disk with the flag off: 13 +~ import work reported by importtime, frozen on: 20150 us +~ import work reported by importtime, frozen off: 22919 us +~ fastest startup with the flag on: 41.1 ms +~ fastest startup with the flag off: 45.6 ms +~ middle startup with the flag on: 52.3 ms +~ middle startup with the flag off: 59.0 ms +~ share of a startup that freezing gives back: 9.9 percent +``` diff --git a/lessons/CLAIMS.md b/lessons/CLAIMS.md index 0d56454..fcae060 100644 --- a/lessons/CLAIMS.md +++ b/lessons/CLAIMS.md @@ -12,7 +12,7 @@ header, what the allocator does with a freed block, the shape of the eval loop. marked with the reason, and a lesson is allowed at most 3 of them. The cap is the point. Without it the exception becomes the rule and this goes back to being a book. -580 claims across 74 lessons, 56 of them not observable from Python. +589 claims across 75 lessons, 57 of them not observable from Python. ## B01. Building CPython, and whether you need to @@ -750,6 +750,20 @@ Without it the exception becomes the rule and this goes back to being a book. | The first import of a module from a source file costs hundreds of times what asking for the same module again costs, and most of that gap survives even once a .pyc file exists | [`r03-22`](r03-what-import-does/r03.ipynb) | | What stops two threads importing two different modules at the same time is the GIL and not the import lock, which a build configured with --disable-gil shows by keeping three and a half cores busy on the same program | not observable from Python: it compares two builds of the same source in two containers, and one notebook cannot be both of them | +## R04. Frozen modules + +| Claim | Proved by | +| --- | --- | +| The module body of importlib._bootstrap contains no IMPORT_NAME opcodes, because the two modules it needs are passed in as arguments and assigned to globals rather than imported | [`r04-07`](r04-frozen-modules/r04.ipynb) | +| A stock 3.15 build has 33 frozen names in three groups, three of which are the import system itself and eleven of which are hello world modules that exist for the test suite | [`r04-10`](r04-frozen-modules/r04.ipynb) | +| A frozen module is a marshalled code object with a co_filename of angle bracket frozen name, and _imp will hand it to you as an ordinary code object you can disassemble | [`r04-13`](r04-frozen-modules/r04.ipynb) | +| A frozen module reports an origin of frozen and a cached of None while still carrying a real path on __file__, which is what lets a traceback through frozen code show you the source line | [`r04-16`](r04-frozen-modules/r04.ipynb) | +| Turning frozen modules off in process removes the standard library and test names from the frozen table and leaves exactly the three bootstrap names, which no setting can remove | [`r04-19`](r04-frozen-modules/r04.ipynb) | +| The same import statement produces a module with an origin of frozen or an origin of a file path depending only on the switch, and the module works identically either way | [`r04-22`](r04-frozen-modules/r04.ipynb) | +| Loading a module from a file and loading it out of the binary end in the same unmarshal of the same bytes, and what freezing removes is the finder search and the file read in front of that | [`r04-25`](r04-frozen-modules/r04.ipynb) | +| Every file a bare startup reads only when frozen modules are switched off is a pyc for a module in the frozen standard library group, which means freezing saves file reading rather than compiling | [`r04-29`](r04-frozen-modules/r04.ipynb) | +| A build configured with --with-pydebug leaves frozen modules off by default while a release build leaves them on, so the same interpreter version can disagree with itself about where os came from | not observable from Python: it needs two builds of the same source in two containers, and one notebook is only ever one of them | + ## T01. One line, seven stages | Claim | Proved by | diff --git a/lessons/README.md b/lessons/README.md index 02cca3c..c2c439a 100644 --- a/lessons/README.md +++ b/lessons/README.md @@ -78,6 +78,7 @@ Each lesson is a notebook you can run. There is nothing to install and nothing t | [R01. Before your first line](r01-before-your-first-line/r01.ipynb) | A child interpreter counting its own sys.modules and splitting them into built into the binary, frozen into the binary and read from a file, with and without site, -X importtime adding up every import that happens before your first line, seven children arguing about sys.flags.optimize to show that the environment and the command line settle it by taking the higher number rather than the nearer one, the whole path configuration printed with an exists check next to every entry so the zip file that is not there stands out, sys.path[0] compared across -c, a script, -m and -P, and ten timed starts of an interpreter that runs nothing at all | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r01-before-your-first-line/r01.ipynb) | | [R02. Where the state lives](r02-where-the-state-lives/r02.ipynb) | Two interpreters in one process asked for the address of the same nine things, a search for the exact int at which they stop agreeing, the interpreter list with its ids counting upwards and never coming back, the recursion limit and the warnings filters and sys.modules changed on one side and read on the other, signal.signal tried from the main thread, from another thread and from inside a second interpreter, two threads each holding a different exception at the same instant, and two recordings pricing an interpreter against an operating system thread on a build with the lock and one without | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r02-where-the-state-lives/r02.ipynb) | | [R03. What import does](r03-what-import-does/r03.ipynb) | The four spellings of the statement compiled and read back opcode by opcode, a finder on the front of sys.meta_path that answers nothing and logs every question a dotted import asks, the three finders asked for the same three names side by side, a circular import in a temporary directory caught reading a module halfway through its own body, a module served out of a string by fourteen lines of class, the three caches an import passes through including the one that hides a directory you just made, a timing of a first import against a repeat one, and two recordings showing that what serialises two threads importing two different modules is the GIL rather than the import lock | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r03-what-import-does/r03.ipynb) | +| [R04. Frozen modules](r04-frozen-modules/r04.ipynb) | The compiled body of the import bootstrap disassembled to show it contains no imports at all, the thirty three frozen names split into the three groups frozen.c keeps them in, a frozen code object pulled straight out of the binary and weighed, os asked five questions about where it came from and answering to one of them and the real path to another, the flag turned off inside a running interpreter and os moving from one loader to another, four steps of a load timed separately to find out which of them freezing actually removes, and a startup with -v counting the files a second interpreter reads that the first one did not | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r04-frozen-modules/r04.ipynb) | ## The three programs diff --git a/lessons/r04-frozen-modules/build.py b/lessons/r04-frozen-modules/build.py new file mode 100644 index 0000000..919eed2 --- /dev/null +++ b/lessons/r04-frozen-modules/build.py @@ -0,0 +1,514 @@ +#!/usr/bin/env python +"""R04. Frozen modules. + +The fourth runtime lesson. R03 finished on a loose end: `import os` never opens `os.py`, because +`FrozenImporter` is asked before `PathFinder` and it says yes. This lesson is about what it is +saying yes to, and about the harder problem underneath, which is that the import system is written +in Python and therefore cannot be imported. + +The method is to read the frozen table from inside a running interpreter. `_imp` exposes all of it: +the list of names, the code objects, and a switch that turns the whole thing off in process. That +switch is what makes the lesson runnable rather than descriptive, because the same import can be +watched arriving both ways without leaving the notebook. + +The two Tier 1 recordings measure what it is worth. The release build and the debug build disagree +about whether to use the frozen copies at all, which is a fact about the build rather than about +Python, and the program reports the default rather than assuming it. + +Run this file to regenerate the notebook, or `just build-lessons` to regenerate all of them. +`just lessons` checks that the committed notebook still matches this file. +""" + +from nbbuild import BANNER, Lesson +from nbdiagram import Diagrams +from tier1 import show as recording + +lesson = Lesson("r04-frozen-modules", "r04") +badge = lesson.badge +cite = lesson.cite +term = lesson.term +figure = Diagrams("r04-frozen-modules").figure + +RELEASE = "r04-what-freezing-saves-at-startup" +DEBUG = "r04-what-freezing-saves-on-a-debug-build" + + +lesson.md(f""" +# R04. Frozen modules + +{badge} + +R03 left a loose end. `import os` never opens `os.py`, because `FrozenImporter` is asked before `PathFinder` and it answers first. This lesson is about what it is answering with. + +Underneath that there is a harder problem. The import system is written in Python, in `Lib/importlib/_bootstrap.py`, which is a file that would have to be imported. Something has to break that circle, and the thing that breaks it is the same thing that makes `import os` skip the disk. + +{figure("from-source-to-binary", "a flow from a source file through a code object and a generated C array into the python3 binary")} +""") + + +lesson.md(""" +## About the source references + +Now and then this lesson points at CPython's own source, like this: `Python/import.c:3389-3428@v3.15.0rc1`. + +Read it as three parts: the file, the lines, and the release those line numbers belong to. Sometimes there is a fourth part after a `#`, which is the name of the thing those lines are inside. + +Every reference is a link, and every one is checked against the pinned source on each change, so a stale reference fails the build instead of sending you somewhere wrong. You never have to read any of it. The references are there so you can go deeper when you want to, and so you can check that this lesson is not making things up. + +## Setup + +Colab does not come with the small package these lessons use, so the next cell installs it. If you are running this from a checkout of the repository it is already installed and the cell does nothing. +""") + + +lesson.code(""" +import sys + +if sys.version_info < (3, 14): + print("This lesson needs CPython 3.14 or newer.") + print(f"This runtime is {sys.version.split()[0]}, and the cells below will not run on it.") +else: + try: + import pyxray + except ImportError: + %pip install -q "pyxray @ git+https://github.com/tamnd/cpython-internals@main#subdirectory=pyxray" + import pyxray +""") + + +lesson.md(""" +## Which Python is this + +Nearly every cell here runs anywhere Python runs, including in a browser tab, because everything the lesson needs is on the `_imp` module and that is compiled into every build. One cell starts a second interpreter to compare two startups, and it says so and skips itself if it cannot. + +## Which interpreter is this +""") + + +lesson.code( + """ +import pyxray + +pyxray.show() +""", + differs=BANNER, + quiet=True, +) + + +lesson.md(f""" +## You cannot import the import system + +`Lib/importlib/_bootstrap.py` is fifteen hundred lines of ordinary Python that implements `__import__`, `sys.meta_path`, module specs and everything else R03 walked through. It is also, obviously, a file. Loading it would need an import, and the thing that performs imports is the file being loaded. + +CPython cuts that loop by compiling the file during its own build and storing the resulting bytecode in the binary as a C array. Loading it at startup is then a read out of memory the loader already has, which is a step the C code can take on its own {cite("Python/import.c:3389-3428@v3.15.0rc1#init_importlib")}. That is a {term("frozen module")}, and it is the whole trick. + +Two things still have to be arranged. The module needs `sys` and `_imp`, and it cannot import them either, so the C code passes both in as arguments to a function called `_setup`, which assigns them to globals {cite("Lib/importlib/_bootstrap.py:1501-1539@v3.15.0rc1#_setup")}. Then `_install` puts `BuiltinImporter` and `FrozenImporter` on `sys.meta_path` and import starts working {cite("Lib/importlib/_bootstrap.py:1541-1546@v3.15.0rc1#_install")}. + +That is the whole {term("import bootstrap")}, and you can check the claim rather than take it: if `_bootstrap.py` really never imports anything at module level, its compiled body has no `IMPORT_NAME` opcodes in it at all. + +{lesson.claim("The module body of importlib._bootstrap contains no IMPORT_NAME opcodes, because the two modules it needs are passed in as arguments and assigned to globals rather than imported")} +""") + + +lesson.code( + """ +import _imp +import dis + +for name in ("_frozen_importlib", "_frozen_importlib_external", "os", "site"): + body = _imp.get_frozen_object(name) + asked = [step for step in dis.get_instructions(body) if step.opname == "IMPORT_NAME"] + print(f" {name:28} {len(asked):2} IMPORT_NAME in the module body") + +print() +print(" the first modules any process has:", list(sys.modules)[:6]) +""", + differs=( + "on 3.14 the two bootstrap counts are the same, but os has 18 rather than 19 and site has " + "7 rather than 11, because more of what they do became an import between the two releases" + ), +) + + +lesson.md(f""" +Zero for the first one, which is the point, and eight for the second one, which is also the point. By the time `_frozen_importlib_external` loads, the module above it has already installed a working import system, so it is allowed to write `import sys` like any other file. + +The `sys.modules` line underneath shows the order. `sys` and `builtins` are put there by the C startup code, then `_frozen_importlib` arrives out of the array, then `_imp` is built by hand and handed over. Nothing on that list was found by searching for anything. + +The name is worth a second look too. The file is `importlib/_bootstrap.py`, but the frozen entry is called `_frozen_importlib`, because the interpreter needs it long before the `importlib` package exists as something you could reach. A table in `Python/frozen.c` maps one name to the other {cite("Python/frozen.c:127-137@v3.15.0rc1#aliases")}, and that {term("module alias")} is how `importlib._bootstrap` and `_frozen_importlib` end up being the same object in `sys.modules`. + +{figure("where-the-loop-is-cut", "a stack of six steps from Py_InitializeFromConfig up to sys.meta_path holding three finders")} + +## What else is in there + +Once the mechanism exists, it is cheap to use it for more than the bootstrap. The list of what gets frozen is a literal table in the build script {cite("Tools/build/freeze_modules.py:37-76@v3.15.0rc1#FROZEN")}, and it lands in the binary as three separate arrays {cite("Python/frozen.c:74-79@v3.15.0rc1#bootstrap_modules")}. + +The three groups are not treated the same way, and `_imp` will show you all of them. + +{lesson.claim("A stock 3.15 build has 33 frozen names in three groups, three of which are the import system itself and eleven of which are hello world modules that exist for the test suite")} +""") + + +lesson.code( + """ +import marshal + +names = _imp._frozen_module_names() +hello = [name for name in names if name.startswith(("__hello", "__phello"))] +stdlib = [name for name in names[3:] if name not in hello] + +print(" names frozen into this binary:", len(names)) +print() +print(" the import system:", ", ".join(names[:3])) +print(" the standard library:", ", ".join(stdlib)) +print(" hello world, for the test suite:", len(hello), "names") +print() +total = sum(len(marshal.dumps(_imp.get_frozen_object(name))) for name in names) +print(f" all of that bytecode, marshalled: {total:,} bytes") +""", + differs=( + "3.14 freezes 28 names rather than 33 and the marshalled total is about 439 thousand bytes " + "rather than 497 thousand, because 3.15 added the encodings package and linecache" + ), +) + + +lesson.md(f""" +{figure("three-groups", "a table of the three frozen arrays, what each holds, how many names it has and whether it can be switched off")} + +The standard library group is everything a bare `python -c pass` touches, which is why it looks like an odd list. It is not the modules people use most. It is the modules that get imported before your first line runs, and R02 counted them. + +The eleven hello world names are `__hello__` and its variations, and they are there so that CPython's own test suite has something to test the machinery on. You can import one right now, and it prints nothing, because the print in it is guarded by `if __name__ == "__main__"`. Try `python -m __hello__` in a terminal to see it speak. + +## What a frozen module is made of + +A frozen entry is a name, a pointer to some bytes, a length and two flags {cite("Python/import.c:3152-3159@v3.15.0rc1#frozen_info")}. The bytes are a marshalled code object, exactly the payload of a {term("pyc file")} without the header. Loading one is a single call to {term("marshal")} over memory that is already there {cite("Python/import.c:3210-3229@v3.15.0rc1#unmarshal_frozen_code")}, and then `exec_module`, which is four lines long {cite("Lib/importlib/_bootstrap.py:1148-1153@v3.15.0rc1#exec_module")}. + +{lesson.claim("A frozen module is a marshalled code object with a co_filename of angle bracket frozen name, and _imp will hand it to you as an ordinary code object you can disassemble")} +""") + + +lesson.code( + """ +body = _imp.get_frozen_object("os") + +print(" what get_frozen_object gives back:", type(body).__name__) +print(" its co_filename: ", body.co_filename) +print(" its co_name: ", body.co_name) +print(f" bytes of bytecode in it: {len(body.co_code):,}") +print() +print(" is_frozen('os'): ", _imp.is_frozen("os")) +packages = [name for name in names if _imp.is_frozen_package(name)] +print(" frozen names that are packages: ", ", ".join(packages)) +print() +for name in ("os", "__phello__.ham", "_frozen_importlib"): + data, is_package, origname = _imp.find_frozen(name) + print(f" find_frozen({name!r:20}) -> package {is_package}, frozen from {origname!r}") +""", + differs=( + "on 3.14 the frozen body of os is 3,088 bytes rather than 3,360, and encodings is not " + "frozen there at all, so the only frozen packages that version has are the hello world ones" + ), +) + + +lesson.md(f""" +`find_frozen` returns three things and the third is the one to notice. For `os` it is just `os`, and for `_frozen_importlib` it is `importlib._bootstrap`, which is the alias table doing its job. The loader keeps that name so it can fix the module up afterwards {cite("Lib/importlib/_bootstrap.py:1000-1011@v3.15.0rc1#FrozenImporter")}. + +`is_package` matters for the same reason it mattered in R03. A frozen package gets an empty `__path__`, which means it is a package with nowhere to look for submodules, so anything underneath one has to be frozen separately under its full dotted name rather than found by searching. `__phello__.ham` is exactly that, and on 3.15 the `encodings` package works the same way, which is why `encodings.utf_8` gets an entry of its own. + +## Frozen, but not hidden + +Here is where it gets interesting. A frozen module never opens a file, and yet it knows perfectly well which file it would have opened. + +`FrozenImporter.find_spec` works out where the source lives, from `sys._stdlib_dir` and the name, and parks it on the spec as {term("loader state")} {cite("Lib/importlib/_bootstrap.py:1106-1133@v3.15.0rc1#find_spec")}. The loader then copies it onto `__file__`. So the spec says `frozen` and `__file__` says `/some/path/os.py`, and both are true. + +{lesson.claim("A frozen module reports an origin of frozen and a cached of None while still carrying a real path on __file__, which is what lets a traceback through frozen code show you the source line")} +""") + + +lesson.code( + """ +import linecache +import os + +print(" os.__spec__.origin: ", os.__spec__.origin) +print(" os.__spec__.cached: ", os.__spec__.cached) +print(" os.__spec__.has_location:", os.__spec__.has_location) +print(" os.__spec__.loader_state:", os.__spec__.loader_state.filename.rpartition("/")[2]) +print(" os.__file__ is a file that exists:", os.path.exists(os.__file__)) +print() +where = os.get_exec_path.__code__ +print(" the code object for os.get_exec_path says it came from:", where.co_filename) +print(" linecache asked for that line with nothing else:") +print(" ", repr(linecache.getline(where.co_filename, where.co_firstlineno))) +print(" and asked again with the module's globals:") +print(" ", repr(linecache.getline(where.co_filename, where.co_firstlineno, os.__dict__))) +""", + varies=( + "the last two lines need the source file to be sitting where the loader state says it is, " + "so a runtime that ships its standard library as a zip with no os.py next to it, which is " + "what a browser does, reports that the path does not exist and gives you two empty strings" + ), +) + + +lesson.md(f""" +{figure("frozen-but-not-hidden", "a table of five questions about the frozen os module and the answer each one gives")} + +That second `linecache` call is the whole trick behind readable tracebacks in frozen code. When the filename starts with `", line 710` and then shows you the actual line. `inspect` has its own version of the same fix up {cite("Lib/inspect.py:3386-3400@v3.15.0rc1#source_file")}. + +The one thing that genuinely is gone is the loader's own source lookup. `FrozenImporter.get_source` returns `None` and always will {cite("Lib/importlib/_bootstrap.py:1155-1165@v3.15.0rc1#get_code")}, because from the loader's point of view there was no source, only bytes. + +## Switching it off + +The bootstrap group has to be frozen or nothing starts. Everything after it is a speed decision, so there is a switch: `-X frozen_modules=off`, or `PYTHON_FROZEN_MODULES=off` in the environment {cite("Python/initconfig.c:2849-2866@v3.15.0rc1#frozen_modules")}. + +The switch cannot reach the bootstrap group, and the reason is four lines of C. `look_up_frozen` walks the bootstrap array unconditionally, and only the standard library and test arrays sit behind a check of the flag {cite("Python/import.c:3104-3129@v3.15.0rc1#look_up_frozen")} {cite("Python/import.c:3130-3150@v3.15.0rc1#use_frozen")}. + +There is also an in process version of the same switch, which the test suite uses and which means you can watch both answers without leaving this notebook. + +{lesson.claim("Turning frozen modules off in process removes the standard library and test names from the frozen table and leaves exactly the three bootstrap names, which no setting can remove")} +""") + + +lesson.code( + """ +from importlib.machinery import FrozenImporter + +print(" frozen names on:", len(_imp._frozen_module_names())) +print(" FrozenImporter.find_spec('stat'):", FrozenImporter.find_spec("stat")) + +_imp._override_frozen_modules_for_tests(-1) +try: + print() + print(" frozen names off:", _imp._frozen_module_names()) + print(" FrozenImporter.find_spec('stat'):", FrozenImporter.find_spec("stat")) + still = FrozenImporter.find_spec("_frozen_importlib") + print(" and the bootstrap is still there:", still.origin) + print(" is_frozen('os') now says:", _imp.is_frozen("os")) +finally: + _imp._override_frozen_modules_for_tests(0) + +print() +print(" back on again, frozen names:", len(_imp._frozen_module_names())) +""", + differs=( + "3.14 freezes 28 names rather than 33, so the first and last lines both read 28 there, and " + "the three names left standing in the middle are the same three on either version" + ), +) + + +lesson.md(f""" +With the switch off, `FrozenImporter` declines to answer for `stat`, so the next finder along gets asked and the import lands on the file. The module you end up with is the same module. Only its paperwork changes. + +{lesson.claim("The same import statement produces a module with an origin of frozen or an origin of a file path depending only on the switch, and the module works identically either way")} +""") + + +lesson.code( + """ +def reimport(name): + \"\"\"Throw a module away and import it again, so a fresh search happens.\"\"\" + sys.modules.pop(name, None) + return __import__(name) + + +_imp._override_frozen_modules_for_tests(-1) +try: + from_disk = reimport("stat") + print(" with the switch off, stat came from:", from_disk.__spec__.origin.rpartition("/")[2]) + print(" its loader: ", type(from_disk.__spec__.loader).__name__) +finally: + _imp._override_frozen_modules_for_tests(0) + +frozen = reimport("stat") +print(" with the switch on, stat came from: ", frozen.__spec__.origin) +print(" its loader: ", frozen.__spec__.loader.__name__) +print() +print(" same constant either way:", from_disk.S_IFREG == frozen.S_IFREG) +""", + varies=( + "the loader you get with the switch off depends on where your runtime keeps its standard " + "library, so a browser, which serves it out of a zip file, reports a zipimporter instead " + "of a SourceFileLoader and a path inside the zip instead of a path on disk" + ), +) + + +lesson.md(f""" +## What one costs to load + +It is tempting to guess that a frozen module loads faster because unmarshalling from memory beats unmarshalling from a file. That is not it. Both paths end in the same `marshal.loads` over the same bytes, and that call is the expensive part. What freezing removes is everything in front of it. + +{figure("two-ways-in", "a comparison of three steps to load a frozen module against five steps to load the same module from disk")} + +{lesson.claim("Loading a module from a file and loading it out of the binary end in the same unmarshal of the same bytes, and what freezing removes is the finder search and the file read in front of that")} +""") + + +lesson.code( + """ +import pathlib +import time +from importlib.machinery import PathFinder + + +def per_call(work, rounds=2000): + \"\"\"Seconds per go, best of three, so one unlucky moment does not decide the answer.\"\"\" + best = None + for _ in range(3): + started = time.perf_counter() + for _ in range(rounds): + work() + taken = (time.perf_counter() - started) / rounds + best = taken if best is None else min(best, taken) + return best + + +asking_frozen = per_call(lambda: FrozenImporter.find_spec("os")) +asking_path = per_call(lambda: PathFinder.find_spec("os", sys.path)) +print(f" asking FrozenImporter for os: {asking_frozen * 1e6:8.2f} microseconds") +print(f" asking PathFinder for the same name: {asking_path * 1e6:8.2f} microseconds") + +tag = f"cpython-{sys.version_info[0]}{sys.version_info[1]}" +pyc = pathlib.Path(os.__file__).with_name("__pycache__") / f"os.{tag}.pyc" +if pyc.exists(): + raw = pyc.read_bytes() + reading = per_call(pyc.read_bytes) + unmarshal = per_call(lambda: marshal.loads(raw[16:])) + print(f" reading those {len(raw):,} bytes off disk: {reading * 1e6:8.2f} microseconds") + print(f" marshal.loads on what it read: {unmarshal * 1e6:8.2f} microseconds") +else: + print(" this runtime keeps no pyc for os, so the last two rows have nothing to measure") +""", + varies=( + "these are timings on your machine, so the absolute numbers depend on your processor and " + "your filesystem, the second row depends on how many entries your sys.path has, and a " + "runtime that keeps its standard library in a zip file has no pyc for os at all" + ), +) + + +lesson.md(""" +The last row is the same work either way, and it is most of the bill. The first two rows are the difference, and they are not close: asking `FrozenImporter` is a lookup in a C array, while asking `PathFinder` means walking `sys.path`, consulting the caches R03 took apart, and finding a match. Add the file read on top of it. + +Multiply that by the number of modules a startup loads before your first line runs, and you have the whole of what freezing is for. + +""") + + +lesson.md(f""" +## What it saves at startup + +Which brings us to the number people actually want. A whole interpreter startup, measured both ways, on a machine that is not otherwise busy. + +The list is the part that travels. Timings depend on your disk and your processor, but which files a startup opens does not, and `-v` prints one line for each code object it reads. + +{lesson.claim("Every file a bare startup reads only when frozen modules are switched off is a pyc for a module in the frozen standard library group, which means freezing saves file reading rather than compiling")} +""") + + +lesson.code( + """ +import subprocess + +ON = ["-X", "frozen_modules=on"] +OFF = ["-X", "frozen_modules=off"] + + +def files_read(flags): + \"\"\"The name of every file a fresh interpreter reads a code object out of, in order.\"\"\" + started = subprocess.run( + [sys.executable, *flags, "-v", "-c", "pass"], capture_output=True, text=True, check=True + ) + lines = started.stderr.splitlines() + return [line.split("'")[1] for line in lines if line.startswith("# code object from")] + + +def module_name(path): + \"\"\"Turn the path of a pyc file back into the name of the module it holds.\"\"\" + parts = pathlib.Path(path).parts + stem = parts[-1].split(".")[0] + return parts[-3] if stem == "__init__" else stem + + +try: + on, off = files_read(ON), files_read(OFF) + print(f" files a bare startup reads with the flag on: {len(on)}, and with it off: {len(off)}") + print() + print(" the ones only the second run needed:") + tails = {name.rpartition(".")[2] for name in names} + for path in [one for one in off if one not in on]: + held = module_name(path) + print(f" {held:20} is in the frozen list: {held in tails}") +except OSError: + print(" this runtime cannot start a second process, so this cell has nothing to show") +""", + varies=( + "the totals depend on what your interpreter loads at startup, and a virtual environment " + "adds a handful of files to both runs, but the list underneath is the frozen standard " + "library either way, minus whichever of those names your version had not frozen yet" + ), +) + + +lesson.md(f""" +Every name on that list is a `.pyc` that had already been compiled, so what freezing saves at startup is not compiling anything. It is a dozen rounds of asking finders, listing directories, opening files and checking timestamps. + +Now the clock. Two Tier 1 recordings run the same program in a container, once on a release build and once on a debug build, and the interesting part is that the two builds disagree about the default. + +{recording(RELEASE)} + +{recording(DEBUG)} + +{figure("what-freezing-saves", "bars comparing startup milliseconds with frozen modules on and off, on a release build and a debug build")} + +Both builds ship the same 33 frozen names, both read 13 files when told not to use them, and both save around a tenth of a startup by using them. What differs is the line that says whether the build uses them unless told otherwise: `True` on the release build and `False` on the debug one. + +That is a deliberate choice in the config defaults {cite("Python/initconfig.c:1193-1201@v3.15.0rc1#use_frozen_modules")}. A debug build is the build you use when you are editing `Lib/os.py`, and it would be miserable if your edits were ignored until you re ran the freeze step. So debug builds read the files, and everybody else gets the speed. + +{lesson.claim("A build configured with --with-pydebug leaves frozen modules off by default while a release build leaves them on, so the same interpreter version can disagree with itself about where os came from", unobservable="it needs two builds of the same source in two containers, and one notebook is only ever one of them")} +""") + + +lesson.md(f""" +## Try it yourself + +Four things, in rough order of how much you will learn. + +Run `python -X frozen_modules=off -X importtime -c pass` and then the same command without the flag, and compare the two trees. Every line that appears only in the first run is a module the frozen build got for free. + +Point the disassembly cell at `_frozen_importlib_external` instead and then open the file at those lines. All eight imports are at the top, written completely normally, because by the time that module loads there is an import system to use. + +Read the last few lines of `Python/frozen.c` {cite("Python/frozen.c:140-144@v3.15.0rc1#PyImport_FrozenModules")}. There is a fourth pointer there, it starts as `NULL`, and the comment invites an embedding application to point it at an array of its own. That is the supported way to ship a single file executable with a standard library inside it, and `look_up_frozen` checks it before the standard library group. + +Delete `stat` from `sys.modules`, turn the switch off, import it, and look at `stat.__spec__.cached`. A file import can write a `.pyc` and a frozen one has nothing to write, so the two runs leave different things behind on your disk. + +## What you now know + +The import system is written in Python and cannot be imported, so it is compiled during CPython's build and stored in the binary as a C array. The C startup code loads it directly and hands it `sys` and `_imp` as arguments. + +`importlib._bootstrap` has no imports in its module body at all. `importlib._bootstrap_external`, which loads second, has eight, because by then import works. + +A frozen module is a marshalled code object and nothing more. Loading one is a read out of memory plus an `exec`. + +There are 33 of them in a stock 3.15 build, in three groups. The three bootstrap names are searched no matter what. The nineteen standard library names and the eleven test names sit behind a flag. + +A frozen module still knows the path it was compiled from, because the loader parks it on the spec and copies it to `__file__`. That is why tracebacks say `` and still show you real source lines. + +`-X frozen_modules=off` is a supported way to see the difference, and `_imp._override_frozen_modules_for_tests` does the same thing without starting a new process. + +Freezing is worth about a tenth of a startup and thirteen files that never get opened. Debug builds leave it off by default so that editing the standard library works the way you expect. + +## What is next + +R05 is about the newest thing in this area. PEP 810 adds lazy imports to 3.15: an import that binds the name now and does the actual loading the first time somebody reads it. + +That changes the shape of everything R03 and R04 established. The module is not in `sys.modules` when the statement finishes, the finders have not been asked yet, and the cost has moved from startup to whenever the name is first touched. The disassembly cell in this lesson already hinted at it, because on 3.15 an `IMPORT_NAME` argument prints with an eager or lazy marker attached to it. +""") + + +raise SystemExit(lesson.save()) diff --git a/lessons/r04-frozen-modules/diagrams.py b/lessons/r04-frozen-modules/diagrams.py new file mode 100644 index 0000000..4b34ce2 --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +"""The diagrams for R04, the modules that live inside the binary. + +Each scene is written out twice, as an editable `.excalidraw` and as the `.svg` the lesson +embeds. Run this file to regenerate them, or `just build-diagrams` for every lesson. + +The order follows the lesson. How a source file becomes part of the binary first, then the +three groups it can land in, then the moment the import system loads itself, then the two +ways a module can arrive, then what is still visible afterwards, then the bill. +""" + +from nbdiagram import Gallery, figures + +gallery = Gallery("r04-frozen-modules") + +gallery.add( + figures.flow( + "from-source-to-binary", + [ + "Lib/os.py", + "a code object", + "os.h, a C array", + "the python3 binary", + ], + title="How a standard library file ends up inside the interpreter", + tones=["input", "intermediate", "intermediate", "durable"], + labels=[ + "compile, during the build", + "marshal", + "the C compiler", + ], + ) +) + + +gallery.add( + figures.table( + "three-groups", + ["the array in frozen.c", "what is in it", "names", "can it be switched off"], + [ + ["bootstrap_modules", "the import system itself", "3", "no"], + ["stdlib_modules", "what a bare startup needs", "19", "yes"], + ["test_modules", "hello world, for the test suite", "11", "yes"], + ], + title="Thirty three frozen names, in three groups that are treated differently", + caption="Only the first group is searched no matter what the flag says.", + tones=["focus", "intermediate", "quiet"], + ) +) + + +gallery.add( + figures.stack( + "where-the-loop-is-cut", + [ + "Py_InitializeFromConfig C, no Python running yet", + "init_importlib still C", + "PyImport_ImportFrozenModule reads _frozen_importlib out of an array", + "_install(sys, _imp) the first Python call of the process", + "_setup(sys, _imp) global sys, _imp, then assign them", + "sys.meta_path three finders, import works from here", + ], + title="Loading the import system without an import system", + note="_bootstrap.py never writes import sys. The module is handed to it as an argument.", + ) +) + + +gallery.add( + figures.compare( + "two-ways-in", + ( + "frozen", + [ + "look the name up in a C array", + "unmarshal the bytes already in memory", + "run the module body", + ], + ), + ( + "the same module from disk", + [ + "ask each finder on sys.meta_path", + "list the directory, look for a match", + "open the pyc, check magic and mtime", + "unmarshal what was read", + "run the module body", + ], + ), + title="What it takes to load os, both ways", + verdict="On a stock build a startup reads 13 of these files with the flag off, none with it on.", + ) +) + + +gallery.add( + figures.table( + "frozen-but-not-hidden", + ["what you ask for", "what you get back"], + [ + ["os.__spec__.origin", "frozen"], + ["os.__spec__.cached", "None, because no pyc was involved"], + ["os.__file__", "the real path to os.py"], + ["os.makedirs.__code__.co_filename", ""], + ["inspect.getsource(os.makedirs)", "the source, found through __file__"], + ], + title="A frozen module knows where it came from, even though it never went there", + caption="The path on the spec is what lets tracebacks and pdb show you real lines.", + tones=["focus", "quiet", "durable", "focus", "durable"], + ) +) + + +gallery.add( + figures.bars( + "what-freezing-saves", + [ + ["release, frozen on", 30.5], + ["release, frozen off", 35.6], + ["debug, frozen on", 41.1], + ["debug, frozen off", 45.6], + ], + unit="ms", + title="Starting up and doing nothing, measured both ways on two builds", + caption="Around a tenth of a startup, and every millisecond of it is file reading.", + tones=["focus", "quiet", "focus", "quiet"], + width=420, + ) +) + + +raise SystemExit(gallery.save()) diff --git a/lessons/r04-frozen-modules/diagrams/from-source-to-binary.excalidraw b/lessons/r04-frozen-modules/diagrams/from-source-to-binary.excalidraw new file mode 100644 index 0000000..b6aa121 --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/from-source-to-binary.excalidraw @@ -0,0 +1,617 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "ae97ec4f0ef2bd0beeb7", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 746.64, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "How a standard library file ends up inside the interpreter", + "originalText": "How a standard library file ends up inside the interpreter", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "3ac3b0a09b6ef7473066", + "type": "rectangle", + "x": 0.0, + "y": 50.0, + "width": 230.0, + "height": 70.0, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "bb53e84e1f68ad0422dd", + "type": "text" + }, + { + "id": "56bb57f2875ce6927179", + "type": "arrow" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "bb53e84e1f68ad0422dd", + "type": "text", + "x": 16.0, + "y": 72.5, + "width": 198.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "Lib/os.py", + "originalText": "Lib/os.py", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "3ac3b0a09b6ef7473066", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "a0ee5e4507279bda0984", + "type": "rectangle", + "x": 0.0, + "y": 180.0, + "width": 230.0, + "height": 70.0, + "angle": 0, + "strokeColor": "#6741d9", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "4bb6383d1d9e199bfc94", + "type": "text" + }, + { + "id": "56bb57f2875ce6927179", + "type": "arrow" + }, + { + "id": "0234da8335fa33fc058d", + "type": "arrow" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "4bb6383d1d9e199bfc94", + "type": "text", + "x": 16.0, + "y": 202.5, + "width": 198.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "a code object", + "originalText": "a code object", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "a0ee5e4507279bda0984", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "39c3205faa0633ace443", + "type": "rectangle", + "x": 0.0, + "y": 310.0, + "width": 230.0, + "height": 70.0, + "angle": 0, + "strokeColor": "#6741d9", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "b1cf309a805a83bf11b4", + "type": "text" + }, + { + "id": "0234da8335fa33fc058d", + "type": "arrow" + }, + { + "id": "0290d1f9b49a670a336d", + "type": "arrow" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "b1cf309a805a83bf11b4", + "type": "text", + "x": 16.0, + "y": 332.5, + "width": 198.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "os.h, a C array", + "originalText": "os.h, a C array", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "39c3205faa0633ace443", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "caa474ce705ff10697dc", + "type": "rectangle", + "x": 0.0, + "y": 440.0, + "width": 230.0, + "height": 70.0, + "angle": 0, + "strokeColor": "#099268", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "ab7a304f56e7dba10317", + "type": "text" + }, + { + "id": "0290d1f9b49a670a336d", + "type": "arrow" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "ab7a304f56e7dba10317", + "type": "text", + "x": 16.0, + "y": 462.5, + "width": 198.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "the python3 binary", + "originalText": "the python3 binary", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "caa474ce705ff10697dc", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "56bb57f2875ce6927179", + "type": "arrow", + "x": 115.0, + "y": 120.0, + "width": 0.0, + "height": 60.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 60.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "3ac3b0a09b6ef7473066", + "focus": 0, + "gap": 4 + }, + "endBinding": { + "elementId": "a0ee5e4507279bda0984", + "focus": 0, + "gap": 4 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "c3421e0f31e1e69245e5", + "type": "text", + "x": 127.0, + "y": 140.0, + "width": 216.95999999999998, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "compile, during the build", + "originalText": "compile, during the build", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "0234da8335fa33fc058d", + "type": "arrow", + "x": 115.0, + "y": 250.0, + "width": 0.0, + "height": 60.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 60.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "a0ee5e4507279bda0984", + "focus": 0, + "gap": 4 + }, + "endBinding": { + "elementId": "39c3205faa0633ace443", + "focus": 0, + "gap": 4 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "34ada453268d638a1886", + "type": "text", + "x": 127.0, + "y": 270.0, + "width": 65.75999999999999, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "marshal", + "originalText": "marshal", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "0290d1f9b49a670a336d", + "type": "arrow", + "x": 115.0, + "y": 380.0, + "width": 0.0, + "height": 60.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 60.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "39c3205faa0633ace443", + "focus": 0, + "gap": 4 + }, + "endBinding": { + "elementId": "caa474ce705ff10697dc", + "focus": 0, + "gap": 4 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "8e7402747a76ce2bb00c", + "type": "text", + "x": 127.0, + "y": 400.0, + "width": 125.75999999999999, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the C compiler", + "originalText": "the C compiler", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r04-frozen-modules/diagrams/from-source-to-binary.svg b/lessons/r04-frozen-modules/diagrams/from-source-to-binary.svg new file mode 100644 index 0000000..df3b1e1 --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/from-source-to-binary.svg @@ -0,0 +1 @@ +How a standard library file ends up inside the interpreterLib/os.pya code objectos.h, a C arraythe python3 binarycompile, during the buildmarshalthe C compiler diff --git a/lessons/r04-frozen-modules/diagrams/frozen-but-not-hidden.excalidraw b/lessons/r04-frozen-modules/diagrams/frozen-but-not-hidden.excalidraw new file mode 100644 index 0000000..569b181 --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/frozen-but-not-hidden.excalidraw @@ -0,0 +1,1013 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "5f26de554797745d0931", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 1016.6399999999999, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "A frozen module knows where it came from, even though it never went there", + "originalText": "A frozen module knows where it came from, even though it never went there", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "09b4d33efcec74f4a6d7", + "type": "text", + "x": 14.0, + "y": 50.0, + "width": 148.56, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "what you ask for", + "originalText": "what you ask for", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "95a741c44fc0d4491bee", + "type": "text", + "x": 349.2, + "y": 50.0, + "width": 161.76, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "what you get back", + "originalText": "what you get back", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "691bdd5f852939c528c7", + "type": "line", + "x": 0.0, + "y": 76.0, + "width": 689.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "c1d8500dd2bad48e7e71", + "type": "line", + "x": 0.0, + "y": 76.0, + "width": 689.5999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ], + [ + 689.5999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "7ea968035528f2a6e008", + "type": "text", + "x": 14.0, + "y": 86.0, + "width": 172.79999999999998, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "os.__spec__.origin", + "originalText": "os.__spec__.origin", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "4057d2ecbf492272eab6", + "type": "text", + "x": 349.2, + "y": 86.0, + "width": 57.599999999999994, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "frozen", + "originalText": "frozen", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "109eb125258e4aa99b42", + "type": "line", + "x": 0.0, + "y": 116.0, + "width": 689.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "b78b6a679d6dafc8c8f8", + "type": "line", + "x": 0.0, + "y": 116.0, + "width": 689.5999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ], + [ + 689.5999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "48c9ed21e5931d0c164c", + "type": "text", + "x": 14.0, + "y": 126.0, + "width": 172.79999999999998, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "os.__spec__.cached", + "originalText": "os.__spec__.cached", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "989ae86fad82ae5690a0", + "type": "text", + "x": 349.2, + "y": 126.0, + "width": 316.8, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "None, because no pyc was involved", + "originalText": "None, because no pyc was involved", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "191dedeff6675c94ff28", + "type": "line", + "x": 0.0, + "y": 156.0, + "width": 689.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "3bb3879993e26d08c9d8", + "type": "line", + "x": 0.0, + "y": 156.0, + "width": 689.5999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ], + [ + 689.5999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "33503536b74fc633db1e", + "type": "text", + "x": 14.0, + "y": 166.0, + "width": 105.6, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "os.__file__", + "originalText": "os.__file__", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "747b893f23720bf5e10f", + "type": "text", + "x": 349.2, + "y": 166.0, + "width": 211.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the real path to os.py", + "originalText": "the real path to os.py", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "22d3f7d35931aae5b9af", + "type": "line", + "x": 0.0, + "y": 196.0, + "width": 689.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "6280286cd1954798a194", + "type": "line", + "x": 0.0, + "y": 196.0, + "width": 689.5999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ], + [ + 689.5999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "7fc4c260fe49ec70002b", + "type": "text", + "x": 14.0, + "y": 206.0, + "width": 307.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "os.makedirs.__code__.co_filename", + "originalText": "os.makedirs.__code__.co_filename", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "bc95043e93d174556e4d", + "type": "text", + "x": 349.2, + "y": 206.0, + "width": 105.6, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "", + "originalText": "", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "07376b662525e61f4c3b", + "type": "line", + "x": 0.0, + "y": 236.0, + "width": 689.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "fb7cad7f3abbceca16e2", + "type": "line", + "x": 0.0, + "y": 236.0, + "width": 689.5999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ], + [ + 689.5999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "6a8d19afd2d80ca843e2", + "type": "text", + "x": 14.0, + "y": 246.0, + "width": 288.0, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "inspect.getsource(os.makedirs)", + "originalText": "inspect.getsource(os.makedirs)", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "72ec39c0158b7b2316a2", + "type": "text", + "x": 349.2, + "y": 246.0, + "width": 326.4, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the source, found through __file__", + "originalText": "the source, found through __file__", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "5e9555deb2fef340f97a", + "type": "line", + "x": 0.0, + "y": 276.0, + "width": 689.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 689.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "45238fb659155ee66e3e", + "type": "text", + "x": 0.0, + "y": 296.0, + "width": 665.52, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "The path on the spec is what lets tracebacks and pdb show you real lines.", + "originalText": "The path on the spec is what lets tracebacks and pdb show you real lines.", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r04-frozen-modules/diagrams/frozen-but-not-hidden.svg b/lessons/r04-frozen-modules/diagrams/frozen-but-not-hidden.svg new file mode 100644 index 0000000..f0c171e --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/frozen-but-not-hidden.svg @@ -0,0 +1 @@ +A frozen module knows where it came from, even though it never went therewhat you ask forwhat you get backos.__spec__.originfrozenos.__spec__.cachedNone, because no pyc was involvedos.__file__the real path to os.pyos.makedirs.__code__.co_filename<frozen os>inspect.getsource(os.makedirs)the source, found through __file__The path on the spec is what lets tracebacks and pdb show you real lines. diff --git a/lessons/r04-frozen-modules/diagrams/three-groups.excalidraw b/lessons/r04-frozen-modules/diagrams/three-groups.excalidraw new file mode 100644 index 0000000..e3f67c8 --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/three-groups.excalidraw @@ -0,0 +1,983 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "e661d14923343f80f455", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 901.4399999999998, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Thirty three frozen names, in three groups that are treated differently", + "originalText": "Thirty three frozen names, in three groups that are treated differently", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "ea9a50481ce058bd4fcc", + "type": "text", + "x": 14.0, + "y": 50.0, + "width": 176.4, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the array in frozen.c", + "originalText": "the array in frozen.c", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "939a4157d798c99dd0ff", + "type": "text", + "x": 243.6, + "y": 50.0, + "width": 112.56, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "what is in it", + "originalText": "what is in it", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "c74665714f5f5921e134", + "type": "text", + "x": 569.1999999999999, + "y": 50.0, + "width": 53.76, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "names", + "originalText": "names", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "b585953fb478847ddfe1", + "type": "text", + "x": 645.1999999999999, + "y": 50.0, + "width": 195.36, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "can it be switched off", + "originalText": "can it be switched off", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "b23d9c36cf06540a1529", + "type": "line", + "x": 0.0, + "y": 76.0, + "width": 870.4, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 870.4, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "b22a5d0f354719456e51", + "type": "line", + "x": 0.0, + "y": 76.0, + "width": 870.4, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 870.4, + 0.0 + ], + [ + 870.4, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "172db0e6d02aae33dcac", + "type": "text", + "x": 14.0, + "y": 86.0, + "width": 163.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "bootstrap_modules", + "originalText": "bootstrap_modules", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "36ce7a3720d199e083ca", + "type": "text", + "x": 243.6, + "y": 86.0, + "width": 230.39999999999998, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the import system itself", + "originalText": "the import system itself", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "e4264aacc22261e8aa85", + "type": "text", + "x": 569.1999999999999, + "y": 86.0, + "width": 9.6, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "3", + "originalText": "3", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "bad9a0653f7352bc0995", + "type": "text", + "x": 645.1999999999999, + "y": 86.0, + "width": 19.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "no", + "originalText": "no", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "ef34787d4bd5eb232d39", + "type": "line", + "x": 0.0, + "y": 116.0, + "width": 870.4, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 870.4, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "144533884dc4552d991b", + "type": "line", + "x": 0.0, + "y": 116.0, + "width": 870.4, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 870.4, + 0.0 + ], + [ + 870.4, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "4f73279c38ea09a62784", + "type": "text", + "x": 14.0, + "y": 126.0, + "width": 134.4, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "stdlib_modules", + "originalText": "stdlib_modules", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "aeaa57d2e1536d2790e8", + "type": "text", + "x": 243.6, + "y": 126.0, + "width": 240.0, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "what a bare startup needs", + "originalText": "what a bare startup needs", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "1b09c58469658399c60a", + "type": "text", + "x": 569.1999999999999, + "y": 126.0, + "width": 19.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "19", + "originalText": "19", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "5dd34681abd6416bb3fb", + "type": "text", + "x": 645.1999999999999, + "y": 126.0, + "width": 28.799999999999997, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "yes", + "originalText": "yes", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "08a0c8fee6d8c06b20cc", + "type": "line", + "x": 0.0, + "y": 156.0, + "width": 870.4, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 870.4, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "756c14849502a2ddbc2c", + "type": "line", + "x": 0.0, + "y": 156.0, + "width": 870.4, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 870.4, + 0.0 + ], + [ + 870.4, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "317b16846071705440ea", + "type": "text", + "x": 14.0, + "y": 166.0, + "width": 115.19999999999999, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "test_modules", + "originalText": "test_modules", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "42562259763b926377bf", + "type": "text", + "x": 243.6, + "y": 166.0, + "width": 297.59999999999997, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "hello world, for the test suite", + "originalText": "hello world, for the test suite", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "af38f17bbd0222e001fb", + "type": "text", + "x": 569.1999999999999, + "y": 166.0, + "width": 19.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "11", + "originalText": "11", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "ff5ef22bec14d75eab2f", + "type": "text", + "x": 645.1999999999999, + "y": 166.0, + "width": 28.799999999999997, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "yes", + "originalText": "yes", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "0a4ff012bf6c465a0aa2", + "type": "line", + "x": 0.0, + "y": 196.0, + "width": 870.4, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 870.4, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "2844d6f67a12b4349bb0", + "type": "text", + "x": 0.0, + "y": 216.0, + "width": 545.52, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Only the first group is searched no matter what the flag says.", + "originalText": "Only the first group is searched no matter what the flag says.", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r04-frozen-modules/diagrams/three-groups.svg b/lessons/r04-frozen-modules/diagrams/three-groups.svg new file mode 100644 index 0000000..9dd14fd --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/three-groups.svg @@ -0,0 +1 @@ +Thirty three frozen names, in three groups that are treated differentlythe array in frozen.cwhat is in itnamescan it be switched offbootstrap_modulesthe import system itself3nostdlib_moduleswhat a bare startup needs19yestest_moduleshello world, for the test suite11yesOnly the first group is searched no matter what the flag says. diff --git a/lessons/r04-frozen-modules/diagrams/two-ways-in.excalidraw b/lessons/r04-frozen-modules/diagrams/two-ways-in.excalidraw new file mode 100644 index 0000000..440cf82 --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/two-ways-in.excalidraw @@ -0,0 +1,753 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "6aaed6d47229be1a9cc4", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 478.08000000000004, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "What it takes to load os, both ways", + "originalText": "What it takes to load os, both ways", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "876c20719876cbe87399", + "type": "text", + "x": 206.5, + "y": 50.0, + "width": 63.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "frozen", + "originalText": "frozen", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "7081725fc342b90740d3", + "type": "rectangle", + "x": 0.0, + "y": 87.0, + "width": 476.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "b8fd6041190385947376", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "b8fd6041190385947376", + "type": "text", + "x": 16.0, + "y": 103.0, + "width": 444.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "look the name up in a C array", + "originalText": "look the name up in a C array", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "7081725fc342b90740d3", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "4582e62819a3c7823bf0", + "type": "rectangle", + "x": 0.0, + "y": 147.0, + "width": 476.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "1c279a76a682d8dbc3b5", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "1c279a76a682d8dbc3b5", + "type": "text", + "x": 16.0, + "y": 163.0, + "width": 444.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "unmarshal the bytes already in memory", + "originalText": "unmarshal the bytes already in memory", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "4582e62819a3c7823bf0", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "9c8e0425e8aa3b71736d", + "type": "rectangle", + "x": 0.0, + "y": 207.0, + "width": 476.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "5173dfe966494443e00c", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "5173dfe966494443e00c", + "type": "text", + "x": 16.0, + "y": 223.0, + "width": 444.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "run the module body", + "originalText": "run the module body", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "9c8e0425e8aa3b71736d", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "9833dd209d9ad9bbea8b", + "type": "text", + "x": 624.45, + "y": 50.0, + "width": 299.09999999999997, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the same module from disk", + "originalText": "the same module from disk", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "e831511cc73d3582acaa", + "type": "rectangle", + "x": 536.0, + "y": 87.0, + "width": 476.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#099268", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "25ca39e3d554a120699c", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "25ca39e3d554a120699c", + "type": "text", + "x": 552.0, + "y": 103.0, + "width": 444.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "ask each finder on sys.meta_path", + "originalText": "ask each finder on sys.meta_path", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "e831511cc73d3582acaa", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "9e2983e49de50054694a", + "type": "rectangle", + "x": 536.0, + "y": 147.0, + "width": 476.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "b7a5b59825ffc94f333d", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "b7a5b59825ffc94f333d", + "type": "text", + "x": 552.0, + "y": 163.0, + "width": 444.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "list the directory, look for a match", + "originalText": "list the directory, look for a match", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "9e2983e49de50054694a", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "38b580644f5e6cd71d8a", + "type": "rectangle", + "x": 536.0, + "y": 207.0, + "width": 476.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "160417f3358a47f97b8f", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "160417f3358a47f97b8f", + "type": "text", + "x": 552.0, + "y": 223.0, + "width": 444.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "open the pyc, check magic and mtime", + "originalText": "open the pyc, check magic and mtime", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "38b580644f5e6cd71d8a", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "8a1060119bab37ef5f33", + "type": "rectangle", + "x": 536.0, + "y": 267.0, + "width": 476.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "2ea02ea8220b9957b46d", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "2ea02ea8220b9957b46d", + "type": "text", + "x": 552.0, + "y": 283.0, + "width": 444.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "unmarshal what was read", + "originalText": "unmarshal what was read", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "8a1060119bab37ef5f33", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "c2d78a076362b25e18f8", + "type": "rectangle", + "x": 536.0, + "y": 327.0, + "width": 476.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "0403ef9883343af30170", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "0403ef9883343af30170", + "type": "text", + "x": 552.0, + "y": 343.0, + "width": 444.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "run the module body", + "originalText": "run the module body", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "c2d78a076362b25e18f8", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "6814da7a2e74087412d5", + "type": "rectangle", + "x": 0.0, + "y": 404.0, + "width": 1012.0, + "height": 60.0, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "a410d849c0ba9cbb0285", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "a410d849c0ba9cbb0285", + "type": "text", + "x": 16.0, + "y": 421.5, + "width": 980.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "On a stock build a startup reads 13 of these files with the flag off, none with it on.", + "originalText": "On a stock build a startup reads 13 of these files with the flag off, none with it on.", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "6814da7a2e74087412d5", + "lineHeight": 1.25, + "autoResize": false + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r04-frozen-modules/diagrams/two-ways-in.svg b/lessons/r04-frozen-modules/diagrams/two-ways-in.svg new file mode 100644 index 0000000..86fcc48 --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/two-ways-in.svg @@ -0,0 +1 @@ +What it takes to load os, both waysfrozenlook the name up in a C arrayunmarshal the bytes already in memoryrun the module bodythe same module from diskask each finder on sys.meta_pathlist the directory, look for a matchopen the pyc, check magic and mtimeunmarshal what was readrun the module bodyOn a stock build a startup reads 13 of these files with the flag off, none with it on. diff --git a/lessons/r04-frozen-modules/diagrams/what-freezing-saves.excalidraw b/lessons/r04-frozen-modules/diagrams/what-freezing-saves.excalidraw new file mode 100644 index 0000000..96577ee --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/what-freezing-saves.excalidraw @@ -0,0 +1,586 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "d48a95d2d1cb0e02605e", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 862.9200000000001, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Starting up and doing nothing, measured both ways on two builds", + "originalText": "Starting up and doing nothing, measured both ways on two builds", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "89b0a04577cc9e0ab047", + "type": "text", + "x": 0.0, + "y": 57.0, + "width": 172.79999999999998, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "release, frozen on", + "originalText": "release, frozen on", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "14af836147c4eb4a1a94", + "type": "line", + "x": 202.4, + "y": 50.0, + "width": 280.92105263157896, + "height": 34.0, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0.0, + 0.0 + ], + [ + 280.92105263157896, + 0.0 + ], + [ + 280.92105263157896, + 34.0 + ], + [ + 0.0, + 34.0 + ], + [ + 0.0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "f5c99f24d070de0ff3ab", + "type": "text", + "x": 493.32105263157894, + "y": 57.0, + "width": 67.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "30.5 ms", + "originalText": "30.5 ms", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "819b56e7dd4617600443", + "type": "text", + "x": 0.0, + "y": 101.0, + "width": 182.4, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "release, frozen off", + "originalText": "release, frozen off", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "7a7cd32d596646683144", + "type": "line", + "x": 202.4, + "y": 94.0, + "width": 327.8947368421053, + "height": 34.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0.0, + 0.0 + ], + [ + 327.8947368421053, + 0.0 + ], + [ + 327.8947368421053, + 34.0 + ], + [ + 0.0, + 34.0 + ], + [ + 0.0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "66213439fac0bee1e255", + "type": "text", + "x": 540.2947368421053, + "y": 101.0, + "width": 67.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "35.6 ms", + "originalText": "35.6 ms", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "82432183ee00dfbbae4d", + "type": "text", + "x": 0.0, + "y": 145.0, + "width": 153.6, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "debug, frozen on", + "originalText": "debug, frozen on", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "7f7a7b1b13723e24ca0b", + "type": "line", + "x": 202.4, + "y": 138.0, + "width": 378.5526315789474, + "height": 34.0, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0.0, + 0.0 + ], + [ + 378.5526315789474, + 0.0 + ], + [ + 378.5526315789474, + 34.0 + ], + [ + 0.0, + 34.0 + ], + [ + 0.0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "a0037eca33a7ee92bdd6", + "type": "text", + "x": 590.9526315789474, + "y": 145.0, + "width": 67.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "41.1 ms", + "originalText": "41.1 ms", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "67ba07c7c2cdd7437d4f", + "type": "text", + "x": 0.0, + "y": 189.0, + "width": 163.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "debug, frozen off", + "originalText": "debug, frozen off", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "b84397f9d363adc98687", + "type": "line", + "x": 202.4, + "y": 182.0, + "width": 420.0, + "height": 34.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0.0, + 0.0 + ], + [ + 420.0, + 0.0 + ], + [ + 420.0, + 34.0 + ], + [ + 0.0, + 34.0 + ], + [ + 0.0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "a1b864eb8c3a84990e87", + "type": "text", + "x": 632.4, + "y": 189.0, + "width": 67.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "45.6 ms", + "originalText": "45.6 ms", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "e1a98eda767589117553", + "type": "text", + "x": 0.0, + "y": 236.0, + "width": 623.76, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Around a tenth of a startup, and every millisecond of it is file reading.", + "originalText": "Around a tenth of a startup, and every millisecond of it is file reading.", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r04-frozen-modules/diagrams/what-freezing-saves.svg b/lessons/r04-frozen-modules/diagrams/what-freezing-saves.svg new file mode 100644 index 0000000..98b92bf --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/what-freezing-saves.svg @@ -0,0 +1 @@ +Starting up and doing nothing, measured both ways on two buildsrelease, frozen on30.5 msrelease, frozen off35.6 msdebug, frozen on41.1 msdebug, frozen off45.6 msAround a tenth of a startup, and every millisecond of it is file reading. diff --git a/lessons/r04-frozen-modules/diagrams/where-the-loop-is-cut.excalidraw b/lessons/r04-frozen-modules/diagrams/where-the-loop-is-cut.excalidraw new file mode 100644 index 0000000..b4bab1f --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/where-the-loop-is-cut.excalidraw @@ -0,0 +1,543 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "e9ccb42bb39f24f4d27b", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 693.0, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Loading the import system without an import system", + "originalText": "Loading the import system without an import system", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "a78f98cf44fafc7a071a", + "type": "rectangle", + "x": 0.0, + "y": 50.0, + "width": 848.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "99f6826979b5c2275ee2", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "99f6826979b5c2275ee2", + "type": "text", + "x": 16.0, + "y": 66.0, + "width": 816.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "sys.meta_path three finders, import works from here", + "originalText": "sys.meta_path three finders, import works from here", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "a78f98cf44fafc7a071a", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "9f3486c4078cb31360c6", + "type": "text", + "x": 864.0, + "y": 66.0, + "width": 25.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "top", + "originalText": "top", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "2472b0958cfd72bd8fdd", + "type": "rectangle", + "x": 0.0, + "y": 102.0, + "width": 848.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#6741d9", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "d1bb0b4f1f662a8eb131", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "d1bb0b4f1f662a8eb131", + "type": "text", + "x": 16.0, + "y": 118.0, + "width": 816.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "_setup(sys, _imp) global sys, _imp, then assign them", + "originalText": "_setup(sys, _imp) global sys, _imp, then assign them", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "2472b0958cfd72bd8fdd", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "27071e2eac9419ac36a2", + "type": "rectangle", + "x": 0.0, + "y": 154.0, + "width": 848.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#6741d9", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "e537bbf010f83b06ea7b", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "e537bbf010f83b06ea7b", + "type": "text", + "x": 16.0, + "y": 170.0, + "width": 816.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "_install(sys, _imp) the first Python call of the process", + "originalText": "_install(sys, _imp) the first Python call of the process", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "27071e2eac9419ac36a2", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "41f6efaeee6ea45a0017", + "type": "rectangle", + "x": 0.0, + "y": 206.0, + "width": 848.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#6741d9", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "caa30e02f5781a1026fc", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "caa30e02f5781a1026fc", + "type": "text", + "x": 16.0, + "y": 222.0, + "width": 816.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "PyImport_ImportFrozenModule reads _frozen_importlib out of an array", + "originalText": "PyImport_ImportFrozenModule reads _frozen_importlib out of an array", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "41f6efaeee6ea45a0017", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "e0b1f63ae6a585fcf708", + "type": "rectangle", + "x": 0.0, + "y": 258.0, + "width": 848.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#6741d9", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "612ba5718c8fba212c45", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "612ba5718c8fba212c45", + "type": "text", + "x": 16.0, + "y": 274.0, + "width": 816.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "init_importlib still C", + "originalText": "init_importlib still C", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "e0b1f63ae6a585fcf708", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "dd85ed5822a46e91234c", + "type": "rectangle", + "x": 0.0, + "y": 310.0, + "width": 848.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#6741d9", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "03d736ef6488030272a4", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "03d736ef6488030272a4", + "type": "text", + "x": 16.0, + "y": 326.0, + "width": 816.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "Py_InitializeFromConfig C, no Python running yet", + "originalText": "Py_InitializeFromConfig C, no Python running yet", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "dd85ed5822a46e91234c", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "bd6c0104ab99d171fcdb", + "type": "text", + "x": 0.0, + "y": 382.0, + "width": 728.6399999999999, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "_bootstrap.py never writes import sys. The module is handed to it as an argument.", + "originalText": "_bootstrap.py never writes import sys. The module is handed to it as an argument.", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r04-frozen-modules/diagrams/where-the-loop-is-cut.svg b/lessons/r04-frozen-modules/diagrams/where-the-loop-is-cut.svg new file mode 100644 index 0000000..91c140d --- /dev/null +++ b/lessons/r04-frozen-modules/diagrams/where-the-loop-is-cut.svg @@ -0,0 +1 @@ +Loading the import system without an import systemsys.meta_path three finders, import works from heretop_setup(sys, _imp) global sys, _imp, then assign them_install(sys, _imp) the first Python call of the processPyImport_ImportFrozenModule reads _frozen_importlib out of an arrayinit_importlib still CPy_InitializeFromConfig C, no Python running yet_bootstrap.py never writes import sys. The module is handed to it as an argument. diff --git a/lessons/r04-frozen-modules/r04.ipynb b/lessons/r04-frozen-modules/r04.ipynb new file mode 100644 index 0000000..1cb09b4 --- /dev/null +++ b/lessons/r04-frozen-modules/r04.ipynb @@ -0,0 +1,932 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "r04-01", + "metadata": {}, + "source": [ + "# R04. Frozen modules\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r04-frozen-modules/r04.ipynb)\n", + "\n", + "R03 left a loose end. `import os` never opens `os.py`, because `FrozenImporter` is asked before `PathFinder` and it answers first. This lesson is about what it is answering with.\n", + "\n", + "Underneath that there is a harder problem. The import system is written in Python, in `Lib/importlib/_bootstrap.py`, which is a file that would have to be imported. Something has to break that circle, and the thing that breaks it is the same thing that makes `import os` skip the disk.\n", + "\n", + "![a flow from a source file through a code object and a generated C array into the python3 binary](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r04-frozen-modules/diagrams/from-source-to-binary.svg)" + ] + }, + { + "cell_type": "markdown", + "id": "r04-02", + "metadata": {}, + "source": [ + "## About the source references\n", + "\n", + "Now and then this lesson points at CPython's own source, like this: `Python/import.c:3389-3428@v3.15.0rc1`.\n", + "\n", + "Read it as three parts: the file, the lines, and the release those line numbers belong to. Sometimes there is a fourth part after a `#`, which is the name of the thing those lines are inside.\n", + "\n", + "Every reference is a link, and every one is checked against the pinned source on each change, so a stale reference fails the build instead of sending you somewhere wrong. You never have to read any of it. The references are there so you can go deeper when you want to, and so you can check that this lesson is not making things up.\n", + "\n", + "## Setup\n", + "\n", + "Colab does not come with the small package these lessons use, so the next cell installs it. If you are running this from a checkout of the repository it is already installed and the cell does nothing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r04-03", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "if sys.version_info < (3, 14):\n", + " print(\"This lesson needs CPython 3.14 or newer.\")\n", + " print(f\"This runtime is {sys.version.split()[0]}, and the cells below will not run on it.\")\n", + "else:\n", + " try:\n", + " import pyxray\n", + " except ImportError:\n", + " %pip install -q \"pyxray @ git+https://github.com/tamnd/cpython-internals@main#subdirectory=pyxray\"\n", + " import pyxray" + ] + }, + { + "cell_type": "markdown", + "id": "r04-04", + "metadata": {}, + "source": [ + "## Which Python is this\n", + "\n", + "Nearly every cell here runs anywhere Python runs, including in a browser tab, because everything the lesson needs is on the `_imp` module and that is compiled into every build. One cell starts a second interpreter to compare two startups, and it says so and skips itself if it cannot.\n", + "\n", + "## Which interpreter is this" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r04-05", + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, + "outputs": [], + "source": [ + "import pyxray\n", + "\n", + "pyxray.show()" + ] + }, + { + "cell_type": "markdown", + "id": "r04-06", + "metadata": {}, + "source": [ + "## You cannot import the import system\n", + "\n", + "`Lib/importlib/_bootstrap.py` is fifteen hundred lines of ordinary Python that implements `__import__`, `sys.meta_path`, module specs and everything else R03 walked through. It is also, obviously, a file. Loading it would need an import, and the thing that performs imports is the file being loaded.\n", + "\n", + "CPython cuts that loop by compiling the file during its own build and storing the resulting bytecode in the binary as a C array. Loading it at startup is then a read out of memory the loader already has, which is a step the C code can take on its own [Python/import.c:3389-3428@v3.15.0rc1#init_importlib](https://github.com/python/cpython/blob/v3.15.0rc1/Python/import.c#L3389-L3428). That is a [frozen module](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#frozen-module), and it is the whole trick.\n", + "\n", + "Two things still have to be arranged. The module needs `sys` and `_imp`, and it cannot import them either, so the C code passes both in as arguments to a function called `_setup`, which assigns them to globals [Lib/importlib/_bootstrap.py:1501-1539@v3.15.0rc1#_setup](https://github.com/python/cpython/blob/v3.15.0rc1/Lib/importlib/_bootstrap.py#L1501-L1539). Then `_install` puts `BuiltinImporter` and `FrozenImporter` on `sys.meta_path` and import starts working [Lib/importlib/_bootstrap.py:1541-1546@v3.15.0rc1#_install](https://github.com/python/cpython/blob/v3.15.0rc1/Lib/importlib/_bootstrap.py#L1541-L1546).\n", + "\n", + "That is the whole [import bootstrap](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#import-bootstrap), and you can check the claim rather than take it: if `_bootstrap.py` really never imports anything at module level, its compiled body has no `IMPORT_NAME` opcodes in it at all.\n", + "\n", + "The module body of importlib._bootstrap contains no IMPORT_NAME opcodes, because the two modules it needs are passed in as arguments and assigned to globals rather than imported" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r04-07", + "metadata": { + "cpython_internals": { + "differs": "on 3.14 the two bootstrap counts are the same, but os has 18 rather than 19 and site has 7 rather than 11, because more of what they do became an import between the two releases" + } + }, + "outputs": [], + "source": [ + "import _imp\n", + "import dis\n", + "\n", + "for name in (\"_frozen_importlib\", \"_frozen_importlib_external\", \"os\", \"site\"):\n", + " body = _imp.get_frozen_object(name)\n", + " asked = [step for step in dis.get_instructions(body) if step.opname == \"IMPORT_NAME\"]\n", + " print(f\" {name:28} {len(asked):2} IMPORT_NAME in the module body\")\n", + "\n", + "print()\n", + "print(\" the first modules any process has:\", list(sys.modules)[:6])" + ] + }, + { + "cell_type": "markdown", + "id": "r04-08", + "metadata": {}, + "source": [ + "> **Version note.** on 3.14 the two bootstrap counts are the same, but os has 18 rather than 19 and site has 7 rather than 11, because more of what they do became an import between the two releases" + ] + }, + { + "cell_type": "markdown", + "id": "r04-09", + "metadata": {}, + "source": [ + "Zero for the first one, which is the point, and eight for the second one, which is also the point. By the time `_frozen_importlib_external` loads, the module above it has already installed a working import system, so it is allowed to write `import sys` like any other file.\n", + "\n", + "The `sys.modules` line underneath shows the order. `sys` and `builtins` are put there by the C startup code, then `_frozen_importlib` arrives out of the array, then `_imp` is built by hand and handed over. Nothing on that list was found by searching for anything.\n", + "\n", + "The name is worth a second look too. The file is `importlib/_bootstrap.py`, but the frozen entry is called `_frozen_importlib`, because the interpreter needs it long before the `importlib` package exists as something you could reach. A table in `Python/frozen.c` maps one name to the other [Python/frozen.c:127-137@v3.15.0rc1#aliases](https://github.com/python/cpython/blob/v3.15.0rc1/Python/frozen.c#L127-L137), and that [module alias](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#module-alias) is how `importlib._bootstrap` and `_frozen_importlib` end up being the same object in `sys.modules`.\n", + "\n", + "![a stack of six steps from Py_InitializeFromConfig up to sys.meta_path holding three finders](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r04-frozen-modules/diagrams/where-the-loop-is-cut.svg)\n", + "\n", + "## What else is in there\n", + "\n", + "Once the mechanism exists, it is cheap to use it for more than the bootstrap. The list of what gets frozen is a literal table in the build script [Tools/build/freeze_modules.py:37-76@v3.15.0rc1#FROZEN](https://github.com/python/cpython/blob/v3.15.0rc1/Tools/build/freeze_modules.py#L37-L76), and it lands in the binary as three separate arrays [Python/frozen.c:74-79@v3.15.0rc1#bootstrap_modules](https://github.com/python/cpython/blob/v3.15.0rc1/Python/frozen.c#L74-L79).\n", + "\n", + "The three groups are not treated the same way, and `_imp` will show you all of them.\n", + "\n", + "A stock 3.15 build has 33 frozen names in three groups, three of which are the import system itself and eleven of which are hello world modules that exist for the test suite" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r04-10", + "metadata": { + "cpython_internals": { + "differs": "3.14 freezes 28 names rather than 33 and the marshalled total is about 439 thousand bytes rather than 497 thousand, because 3.15 added the encodings package and linecache" + } + }, + "outputs": [], + "source": [ + "import marshal\n", + "\n", + "names = _imp._frozen_module_names()\n", + "hello = [name for name in names if name.startswith((\"__hello\", \"__phello\"))]\n", + "stdlib = [name for name in names[3:] if name not in hello]\n", + "\n", + "print(\" names frozen into this binary:\", len(names))\n", + "print()\n", + "print(\" the import system:\", \", \".join(names[:3]))\n", + "print(\" the standard library:\", \", \".join(stdlib))\n", + "print(\" hello world, for the test suite:\", len(hello), \"names\")\n", + "print()\n", + "total = sum(len(marshal.dumps(_imp.get_frozen_object(name))) for name in names)\n", + "print(f\" all of that bytecode, marshalled: {total:,} bytes\")" + ] + }, + { + "cell_type": "markdown", + "id": "r04-11", + "metadata": {}, + "source": [ + "> **Version note.** 3.14 freezes 28 names rather than 33 and the marshalled total is about 439 thousand bytes rather than 497 thousand, because 3.15 added the encodings package and linecache" + ] + }, + { + "cell_type": "markdown", + "id": "r04-12", + "metadata": {}, + "source": [ + "![a table of the three frozen arrays, what each holds, how many names it has and whether it can be switched off](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r04-frozen-modules/diagrams/three-groups.svg)\n", + "\n", + "The standard library group is everything a bare `python -c pass` touches, which is why it looks like an odd list. It is not the modules people use most. It is the modules that get imported before your first line runs, and R02 counted them.\n", + "\n", + "The eleven hello world names are `__hello__` and its variations, and they are there so that CPython's own test suite has something to test the machinery on. You can import one right now, and it prints nothing, because the print in it is guarded by `if __name__ == \"__main__\"`. Try `python -m __hello__` in a terminal to see it speak.\n", + "\n", + "## What a frozen module is made of\n", + "\n", + "A frozen entry is a name, a pointer to some bytes, a length and two flags [Python/import.c:3152-3159@v3.15.0rc1#frozen_info](https://github.com/python/cpython/blob/v3.15.0rc1/Python/import.c#L3152-L3159). The bytes are a marshalled code object, exactly the payload of a [pyc file](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#pyc-file) without the header. Loading one is a single call to [marshal](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#marshal) over memory that is already there [Python/import.c:3210-3229@v3.15.0rc1#unmarshal_frozen_code](https://github.com/python/cpython/blob/v3.15.0rc1/Python/import.c#L3210-L3229), and then `exec_module`, which is four lines long [Lib/importlib/_bootstrap.py:1148-1153@v3.15.0rc1#exec_module](https://github.com/python/cpython/blob/v3.15.0rc1/Lib/importlib/_bootstrap.py#L1148-L1153).\n", + "\n", + "A frozen module is a marshalled code object with a co_filename of angle bracket frozen name, and _imp will hand it to you as an ordinary code object you can disassemble" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r04-13", + "metadata": { + "cpython_internals": { + "differs": "on 3.14 the frozen body of os is 3,088 bytes rather than 3,360, and encodings is not frozen there at all, so the only frozen packages that version has are the hello world ones" + } + }, + "outputs": [], + "source": [ + "body = _imp.get_frozen_object(\"os\")\n", + "\n", + "print(\" what get_frozen_object gives back:\", type(body).__name__)\n", + "print(\" its co_filename: \", body.co_filename)\n", + "print(\" its co_name: \", body.co_name)\n", + "print(f\" bytes of bytecode in it: {len(body.co_code):,}\")\n", + "print()\n", + "print(\" is_frozen('os'): \", _imp.is_frozen(\"os\"))\n", + "packages = [name for name in names if _imp.is_frozen_package(name)]\n", + "print(\" frozen names that are packages: \", \", \".join(packages))\n", + "print()\n", + "for name in (\"os\", \"__phello__.ham\", \"_frozen_importlib\"):\n", + " data, is_package, origname = _imp.find_frozen(name)\n", + " print(f\" find_frozen({name!r:20}) -> package {is_package}, frozen from {origname!r}\")" + ] + }, + { + "cell_type": "markdown", + "id": "r04-14", + "metadata": {}, + "source": [ + "> **Version note.** on 3.14 the frozen body of os is 3,088 bytes rather than 3,360, and encodings is not frozen there at all, so the only frozen packages that version has are the hello world ones" + ] + }, + { + "cell_type": "markdown", + "id": "r04-15", + "metadata": {}, + "source": [ + "`find_frozen` returns three things and the third is the one to notice. For `os` it is just `os`, and for `_frozen_importlib` it is `importlib._bootstrap`, which is the alias table doing its job. The loader keeps that name so it can fix the module up afterwards [Lib/importlib/_bootstrap.py:1000-1011@v3.15.0rc1#FrozenImporter](https://github.com/python/cpython/blob/v3.15.0rc1/Lib/importlib/_bootstrap.py#L1000-L1011).\n", + "\n", + "`is_package` matters for the same reason it mattered in R03. A frozen package gets an empty `__path__`, which means it is a package with nowhere to look for submodules, so anything underneath one has to be frozen separately under its full dotted name rather than found by searching. `__phello__.ham` is exactly that, and on 3.15 the `encodings` package works the same way, which is why `encodings.utf_8` gets an entry of its own.\n", + "\n", + "## Frozen, but not hidden\n", + "\n", + "Here is where it gets interesting. A frozen module never opens a file, and yet it knows perfectly well which file it would have opened.\n", + "\n", + "`FrozenImporter.find_spec` works out where the source lives, from `sys._stdlib_dir` and the name, and parks it on the spec as [loader state](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#loader-state) [Lib/importlib/_bootstrap.py:1106-1133@v3.15.0rc1#find_spec](https://github.com/python/cpython/blob/v3.15.0rc1/Lib/importlib/_bootstrap.py#L1106-L1133). The loader then copies it onto `__file__`. So the spec says `frozen` and `__file__` says `/some/path/os.py`, and both are true.\n", + "\n", + "A frozen module reports an origin of frozen and a cached of None while still carrying a real path on __file__, which is what lets a traceback through frozen code show you the source line" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r04-16", + "metadata": { + "cpython_internals": { + "varies": "the last two lines need the source file to be sitting where the loader state says it is, so a runtime that ships its standard library as a zip with no os.py next to it, which is what a browser does, reports that the path does not exist and gives you two empty strings" + } + }, + "outputs": [], + "source": [ + "import linecache\n", + "import os\n", + "\n", + "print(\" os.__spec__.origin: \", os.__spec__.origin)\n", + "print(\" os.__spec__.cached: \", os.__spec__.cached)\n", + "print(\" os.__spec__.has_location:\", os.__spec__.has_location)\n", + "print(\" os.__spec__.loader_state:\", os.__spec__.loader_state.filename.rpartition(\"/\")[2])\n", + "print(\" os.__file__ is a file that exists:\", os.path.exists(os.__file__))\n", + "print()\n", + "where = os.get_exec_path.__code__\n", + "print(\" the code object for os.get_exec_path says it came from:\", where.co_filename)\n", + "print(\" linecache asked for that line with nothing else:\")\n", + "print(\" \", repr(linecache.getline(where.co_filename, where.co_firstlineno)))\n", + "print(\" and asked again with the module's globals:\")\n", + "print(\" \", repr(linecache.getline(where.co_filename, where.co_firstlineno, os.__dict__)))" + ] + }, + { + "cell_type": "markdown", + "id": "r04-17", + "metadata": {}, + "source": [ + "> **Version note.** the last two lines need the source file to be sitting where the loader state says it is, so a runtime that ships its standard library as a zip with no os.py next to it, which is what a browser does, reports that the path does not exist and gives you two empty strings" + ] + }, + { + "cell_type": "markdown", + "id": "r04-18", + "metadata": {}, + "source": [ + "![a table of five questions about the frozen os module and the answer each one gives](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r04-frozen-modules/diagrams/frozen-but-not-hidden.svg)\n", + "\n", + "That second `linecache` call is the whole trick behind readable tracebacks in frozen code. When the filename starts with `\", line 710` and then shows you the actual line. `inspect` has its own version of the same fix up [Lib/inspect.py:3386-3400@v3.15.0rc1#source_file](https://github.com/python/cpython/blob/v3.15.0rc1/Lib/inspect.py#L3386-L3400).\n", + "\n", + "The one thing that genuinely is gone is the loader's own source lookup. `FrozenImporter.get_source` returns `None` and always will [Lib/importlib/_bootstrap.py:1155-1165@v3.15.0rc1#get_code](https://github.com/python/cpython/blob/v3.15.0rc1/Lib/importlib/_bootstrap.py#L1155-L1165), because from the loader's point of view there was no source, only bytes.\n", + "\n", + "## Switching it off\n", + "\n", + "The bootstrap group has to be frozen or nothing starts. Everything after it is a speed decision, so there is a switch: `-X frozen_modules=off`, or `PYTHON_FROZEN_MODULES=off` in the environment [Python/initconfig.c:2849-2866@v3.15.0rc1#frozen_modules](https://github.com/python/cpython/blob/v3.15.0rc1/Python/initconfig.c#L2849-L2866).\n", + "\n", + "The switch cannot reach the bootstrap group, and the reason is four lines of C. `look_up_frozen` walks the bootstrap array unconditionally, and only the standard library and test arrays sit behind a check of the flag [Python/import.c:3104-3129@v3.15.0rc1#look_up_frozen](https://github.com/python/cpython/blob/v3.15.0rc1/Python/import.c#L3104-L3129) [Python/import.c:3130-3150@v3.15.0rc1#use_frozen](https://github.com/python/cpython/blob/v3.15.0rc1/Python/import.c#L3130-L3150).\n", + "\n", + "There is also an in process version of the same switch, which the test suite uses and which means you can watch both answers without leaving this notebook.\n", + "\n", + "Turning frozen modules off in process removes the standard library and test names from the frozen table and leaves exactly the three bootstrap names, which no setting can remove" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r04-19", + "metadata": { + "cpython_internals": { + "differs": "3.14 freezes 28 names rather than 33, so the first and last lines both read 28 there, and the three names left standing in the middle are the same three on either version" + } + }, + "outputs": [], + "source": [ + "from importlib.machinery import FrozenImporter\n", + "\n", + "print(\" frozen names on:\", len(_imp._frozen_module_names()))\n", + "print(\" FrozenImporter.find_spec('stat'):\", FrozenImporter.find_spec(\"stat\"))\n", + "\n", + "_imp._override_frozen_modules_for_tests(-1)\n", + "try:\n", + " print()\n", + " print(\" frozen names off:\", _imp._frozen_module_names())\n", + " print(\" FrozenImporter.find_spec('stat'):\", FrozenImporter.find_spec(\"stat\"))\n", + " still = FrozenImporter.find_spec(\"_frozen_importlib\")\n", + " print(\" and the bootstrap is still there:\", still.origin)\n", + " print(\" is_frozen('os') now says:\", _imp.is_frozen(\"os\"))\n", + "finally:\n", + " _imp._override_frozen_modules_for_tests(0)\n", + "\n", + "print()\n", + "print(\" back on again, frozen names:\", len(_imp._frozen_module_names()))" + ] + }, + { + "cell_type": "markdown", + "id": "r04-20", + "metadata": {}, + "source": [ + "> **Version note.** 3.14 freezes 28 names rather than 33, so the first and last lines both read 28 there, and the three names left standing in the middle are the same three on either version" + ] + }, + { + "cell_type": "markdown", + "id": "r04-21", + "metadata": {}, + "source": [ + "With the switch off, `FrozenImporter` declines to answer for `stat`, so the next finder along gets asked and the import lands on the file. The module you end up with is the same module. Only its paperwork changes.\n", + "\n", + "The same import statement produces a module with an origin of frozen or an origin of a file path depending only on the switch, and the module works identically either way" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r04-22", + "metadata": { + "cpython_internals": { + "varies": "the loader you get with the switch off depends on where your runtime keeps its standard library, so a browser, which serves it out of a zip file, reports a zipimporter instead of a SourceFileLoader and a path inside the zip instead of a path on disk" + } + }, + "outputs": [], + "source": [ + "def reimport(name):\n", + " \"\"\"Throw a module away and import it again, so a fresh search happens.\"\"\"\n", + " sys.modules.pop(name, None)\n", + " return __import__(name)\n", + "\n", + "\n", + "_imp._override_frozen_modules_for_tests(-1)\n", + "try:\n", + " from_disk = reimport(\"stat\")\n", + " print(\" with the switch off, stat came from:\", from_disk.__spec__.origin.rpartition(\"/\")[2])\n", + " print(\" its loader: \", type(from_disk.__spec__.loader).__name__)\n", + "finally:\n", + " _imp._override_frozen_modules_for_tests(0)\n", + "\n", + "frozen = reimport(\"stat\")\n", + "print(\" with the switch on, stat came from: \", frozen.__spec__.origin)\n", + "print(\" its loader: \", frozen.__spec__.loader.__name__)\n", + "print()\n", + "print(\" same constant either way:\", from_disk.S_IFREG == frozen.S_IFREG)" + ] + }, + { + "cell_type": "markdown", + "id": "r04-23", + "metadata": {}, + "source": [ + "> **Version note.** the loader you get with the switch off depends on where your runtime keeps its standard library, so a browser, which serves it out of a zip file, reports a zipimporter instead of a SourceFileLoader and a path inside the zip instead of a path on disk" + ] + }, + { + "cell_type": "markdown", + "id": "r04-24", + "metadata": {}, + "source": [ + "## What one costs to load\n", + "\n", + "It is tempting to guess that a frozen module loads faster because unmarshalling from memory beats unmarshalling from a file. That is not it. Both paths end in the same `marshal.loads` over the same bytes, and that call is the expensive part. What freezing removes is everything in front of it.\n", + "\n", + "![a comparison of three steps to load a frozen module against five steps to load the same module from disk](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r04-frozen-modules/diagrams/two-ways-in.svg)\n", + "\n", + "Loading a module from a file and loading it out of the binary end in the same unmarshal of the same bytes, and what freezing removes is the finder search and the file read in front of that" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r04-25", + "metadata": { + "cpython_internals": { + "varies": "these are timings on your machine, so the absolute numbers depend on your processor and your filesystem, the second row depends on how many entries your sys.path has, and a runtime that keeps its standard library in a zip file has no pyc for os at all" + } + }, + "outputs": [], + "source": [ + "import pathlib\n", + "import time\n", + "from importlib.machinery import PathFinder\n", + "\n", + "\n", + "def per_call(work, rounds=2000):\n", + " \"\"\"Seconds per go, best of three, so one unlucky moment does not decide the answer.\"\"\"\n", + " best = None\n", + " for _ in range(3):\n", + " started = time.perf_counter()\n", + " for _ in range(rounds):\n", + " work()\n", + " taken = (time.perf_counter() - started) / rounds\n", + " best = taken if best is None else min(best, taken)\n", + " return best\n", + "\n", + "\n", + "asking_frozen = per_call(lambda: FrozenImporter.find_spec(\"os\"))\n", + "asking_path = per_call(lambda: PathFinder.find_spec(\"os\", sys.path))\n", + "print(f\" asking FrozenImporter for os: {asking_frozen * 1e6:8.2f} microseconds\")\n", + "print(f\" asking PathFinder for the same name: {asking_path * 1e6:8.2f} microseconds\")\n", + "\n", + "tag = f\"cpython-{sys.version_info[0]}{sys.version_info[1]}\"\n", + "pyc = pathlib.Path(os.__file__).with_name(\"__pycache__\") / f\"os.{tag}.pyc\"\n", + "if pyc.exists():\n", + " raw = pyc.read_bytes()\n", + " reading = per_call(pyc.read_bytes)\n", + " unmarshal = per_call(lambda: marshal.loads(raw[16:]))\n", + " print(f\" reading those {len(raw):,} bytes off disk: {reading * 1e6:8.2f} microseconds\")\n", + " print(f\" marshal.loads on what it read: {unmarshal * 1e6:8.2f} microseconds\")\n", + "else:\n", + " print(\" this runtime keeps no pyc for os, so the last two rows have nothing to measure\")" + ] + }, + { + "cell_type": "markdown", + "id": "r04-26", + "metadata": {}, + "source": [ + "> **Version note.** these are timings on your machine, so the absolute numbers depend on your processor and your filesystem, the second row depends on how many entries your sys.path has, and a runtime that keeps its standard library in a zip file has no pyc for os at all" + ] + }, + { + "cell_type": "markdown", + "id": "r04-27", + "metadata": {}, + "source": [ + "The last row is the same work either way, and it is most of the bill. The first two rows are the difference, and they are not close: asking `FrozenImporter` is a lookup in a C array, while asking `PathFinder` means walking `sys.path`, consulting the caches R03 took apart, and finding a match. Add the file read on top of it.\n", + "\n", + "Multiply that by the number of modules a startup loads before your first line runs, and you have the whole of what freezing is for." + ] + }, + { + "cell_type": "markdown", + "id": "r04-28", + "metadata": {}, + "source": [ + "## What it saves at startup\n", + "\n", + "Which brings us to the number people actually want. A whole interpreter startup, measured both ways, on a machine that is not otherwise busy.\n", + "\n", + "The list is the part that travels. Timings depend on your disk and your processor, but which files a startup opens does not, and `-v` prints one line for each code object it reads.\n", + "\n", + "Every file a bare startup reads only when frozen modules are switched off is a pyc for a module in the frozen standard library group, which means freezing saves file reading rather than compiling" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r04-29", + "metadata": { + "cpython_internals": { + "varies": "the totals depend on what your interpreter loads at startup, and a virtual environment adds a handful of files to both runs, but the list underneath is the frozen standard library either way, minus whichever of those names your version had not frozen yet" + } + }, + "outputs": [], + "source": [ + "import subprocess\n", + "\n", + "ON = [\"-X\", \"frozen_modules=on\"]\n", + "OFF = [\"-X\", \"frozen_modules=off\"]\n", + "\n", + "\n", + "def files_read(flags):\n", + " \"\"\"The name of every file a fresh interpreter reads a code object out of, in order.\"\"\"\n", + " started = subprocess.run(\n", + " [sys.executable, *flags, \"-v\", \"-c\", \"pass\"], capture_output=True, text=True, check=True\n", + " )\n", + " lines = started.stderr.splitlines()\n", + " return [line.split(\"'\")[1] for line in lines if line.startswith(\"# code object from\")]\n", + "\n", + "\n", + "def module_name(path):\n", + " \"\"\"Turn the path of a pyc file back into the name of the module it holds.\"\"\"\n", + " parts = pathlib.Path(path).parts\n", + " stem = parts[-1].split(\".\")[0]\n", + " return parts[-3] if stem == \"__init__\" else stem\n", + "\n", + "\n", + "try:\n", + " on, off = files_read(ON), files_read(OFF)\n", + " print(f\" files a bare startup reads with the flag on: {len(on)}, and with it off: {len(off)}\")\n", + " print()\n", + " print(\" the ones only the second run needed:\")\n", + " tails = {name.rpartition(\".\")[2] for name in names}\n", + " for path in [one for one in off if one not in on]:\n", + " held = module_name(path)\n", + " print(f\" {held:20} is in the frozen list: {held in tails}\")\n", + "except OSError:\n", + " print(\" this runtime cannot start a second process, so this cell has nothing to show\")" + ] + }, + { + "cell_type": "markdown", + "id": "r04-30", + "metadata": {}, + "source": [ + "> **Version note.** the totals depend on what your interpreter loads at startup, and a virtual environment adds a handful of files to both runs, but the list underneath is the frozen standard library either way, minus whichever of those names your version had not frozen yet" + ] + }, + { + "cell_type": "markdown", + "id": "r04-31", + "metadata": {}, + "source": [ + "Every name on that list is a `.pyc` that had already been compiled, so what freezing saves at startup is not compiling anything. It is a dozen rounds of asking finders, listing directories, opening files and checking timestamps.\n", + "\n", + "Now the clock. Two Tier 1 recordings run the same program in a container, once on a release build and once on a debug build, and the interesting part is that the two builds disagree about the default.\n", + "\n", + "What does compiling the standard library into the binary actually save at startup?\n", + "\n", + "```python\n", + "\"\"\"What freezing the standard library into the binary is worth at startup.\n", + "\n", + "Import is written in Python, so it cannot be imported. CPython gets around that by compiling a\n", + "handful of modules during its own build and writing the bytecode into the binary as C arrays. The\n", + "first three are the import system itself, and they are what lets the interpreter get going at all.\n", + "\n", + "Everything after those three is a speed decision rather than a correctness one, and it can be\n", + "switched off with -X frozen_modules=off, so the cost of it can be measured rather than guessed.\n", + "Which way the switch sits by default is a build choice, so this program never relies on the\n", + "default. It asks for on and off explicitly and reports what the default happens to be.\n", + "\n", + "Two numbers matter. The first is how many code objects a bare startup reads off disk, which -v\n", + "prints one line for, so it is a count rather than a timing and it is the same on any machine. The\n", + "second is wall clock, measured with the two cases alternating, because anything that runs one case\n", + "forty times and then the other forty times is measuring the state of the page cache instead.\n", + "\"\"\"\n", + "\n", + "import _imp\n", + "import statistics\n", + "import subprocess\n", + "import sys\n", + "import time\n", + "\n", + "ROUNDS = 40\n", + "ON = [\"-X\", \"frozen_modules=on\"]\n", + "OFF = [\"-X\", \"frozen_modules=off\"]\n", + "ORIGIN = \"import os; print(os.__spec__.origin)\"\n", + "BOOTSTRAP = \"import sys; print(sys.modules['_frozen_importlib'].__spec__.origin)\"\n", + "COUNT = (\n", + " \"import sys\\n\"\n", + " \"specs = [getattr(m, '__spec__', None) for m in sys.modules.values()]\\n\"\n", + " \"print(len(sys.modules), sum(1 for s in specs if s is not None and s.origin == 'frozen'))\\n\"\n", + ")\n", + "\n", + "\n", + "def child(flags, args):\n", + " \"\"\"Run this same interpreter again with those flags and hand back the finished process.\"\"\"\n", + " return subprocess.run(\n", + " [sys.executable, *flags, *args], capture_output=True, text=True, check=True\n", + " )\n", + "\n", + "\n", + "def origin(flags):\n", + " \"\"\"Ask a fresh interpreter where the os module it just imported came from.\"\"\"\n", + " return child(flags, [\"-c\", ORIGIN]).stdout.strip()\n", + "\n", + "\n", + "def loaded(flags):\n", + " \"\"\"Ask a fresh interpreter how many modules it loaded and how many came out of the binary.\"\"\"\n", + " return [int(part) for part in child(flags, [\"-c\", COUNT]).stdout.split()]\n", + "\n", + "\n", + "def files_read(flags):\n", + " \"\"\"Count the code objects a fresh interpreter reads off disk, which -v prints one per line.\"\"\"\n", + " printed = child(flags, [\"-v\", \"-c\", \"pass\"]).stderr\n", + " return sum(1 for line in printed.splitlines() if line.startswith(\"# code object from\"))\n", + "\n", + "\n", + "def import_work(flags):\n", + " \"\"\"Add up the self time -X importtime reports, in microseconds.\"\"\"\n", + " printed = child([\"-X\", \"importtime\", *flags], [\"-c\", \"pass\"]).stderr\n", + " total = 0\n", + " for line in printed.splitlines():\n", + " if line.startswith(\"import time:\"):\n", + " first = line.removeprefix(\"import time:\").split(\"|\")[0].strip()\n", + " if first.isdigit():\n", + " total += int(first)\n", + " return total\n", + "\n", + "\n", + "def startup_ms(flags):\n", + " \"\"\"Wall clock milliseconds for one whole interpreter startup that does nothing at all.\"\"\"\n", + " started = time.perf_counter()\n", + " child(flags, [\"-c\", \"pass\"])\n", + " return (time.perf_counter() - started) * 1000\n", + "\n", + "\n", + "def alternating(measure, rounds):\n", + " \"\"\"Measure both cases once each per round, so neither one gets the cold cache every time.\"\"\"\n", + " got = {\"on\": [], \"off\": []}\n", + " for _ in range(rounds):\n", + " got[\"on\"].append(measure(ON))\n", + " got[\"off\"].append(measure(OFF))\n", + " return got\n", + "\n", + "\n", + "_imp._override_frozen_modules_for_tests(1)\n", + "names = _imp._frozen_module_names()\n", + "frozen_os = _imp.get_frozen_object(\"os\")\n", + "_imp._override_frozen_modules_for_tests(0)\n", + "\n", + "print(\"names compiled into this binary:\", len(names))\n", + "print(\"the three that cannot be turned off:\", \", \".join(names[:3]))\n", + "print(\"filename on the frozen code object for os:\", frozen_os.co_filename)\n", + "print(\"bytes of bytecode in it:\", len(frozen_os.co_code))\n", + "print(\"this build uses them unless told otherwise:\", origin([]) == \"frozen\")\n", + "print(\"where os comes from with the flag on:\", origin(ON))\n", + "print(\"where os comes from with the flag off:\", origin(OFF).rpartition(\"/\")[2])\n", + "print(\"_frozen_importlib with the flag off:\", child(OFF, [\"-c\", BOOTSTRAP]).stdout.strip())\n", + "\n", + "on_total, on_frozen = loaded(ON)\n", + "off_total, off_frozen = loaded(OFF)\n", + "print(\"modules a bare startup loads, flag on:\", on_total)\n", + "print(\"modules a bare startup loads, flag off:\", off_total)\n", + "print(\"of those, frozen with the flag on:\", on_frozen)\n", + "print(\"of those, frozen with the flag off:\", off_frozen)\n", + "print(\"code objects read off disk with the flag on:\", files_read(ON))\n", + "print(\"code objects read off disk with the flag off:\", files_read(OFF))\n", + "\n", + "work = alternating(import_work, 10)\n", + "print(f\"~ import work reported by importtime, frozen on: {min(work['on'])} us\")\n", + "print(f\"~ import work reported by importtime, frozen off: {min(work['off'])} us\")\n", + "\n", + "runs = alternating(startup_ms, ROUNDS)\n", + "fast_on, fast_off = min(runs[\"on\"]), min(runs[\"off\"])\n", + "print(f\"~ fastest startup with the flag on: {fast_on:.1f} ms\")\n", + "print(f\"~ fastest startup with the flag off: {fast_off:.1f} ms\")\n", + "print(f\"~ middle startup with the flag on: {statistics.median(runs['on']):.1f} ms\")\n", + "print(f\"~ middle startup with the flag off: {statistics.median(runs['off']):.1f} ms\")\n", + "share = (1 - fast_on / fast_off) * 100\n", + "print(f\"~ share of a startup that freezing gives back: {share:.1f} percent\")\n", + "```\n", + "\n", + "```text\n", + "names compiled into this binary: 33\n", + "the three that cannot be turned off: _frozen_importlib, _frozen_importlib_external, zipimport\n", + "filename on the frozen code object for os: \n", + "bytes of bytecode in it: 3360\n", + "this build uses them unless told otherwise: True\n", + "where os comes from with the flag on: frozen\n", + "where os comes from with the flag off: os.py\n", + "_frozen_importlib with the flag off: frozen\n", + "modules a bare startup loads, flag on: 33\n", + "modules a bare startup loads, flag off: 33\n", + "of those, frozen with the flag on: 17\n", + "of those, frozen with the flag off: 3\n", + "code objects read off disk with the flag on: 0\n", + "code objects read off disk with the flag off: 13\n", + "~ import work reported by importtime, frozen on: 12884 us\n", + "~ import work reported by importtime, frozen off: 19167 us\n", + "~ fastest startup with the flag on: 30.5 ms\n", + "~ fastest startup with the flag off: 35.6 ms\n", + "~ middle startup with the flag on: 33.9 ms\n", + "~ middle startup with the flag off: 39.5 ms\n", + "~ share of a startup that freezing gives back: 14.2 percent\n", + "```\n", + "\n", + "That ran on Python 3.15.0rc1 in the release build this project publishes, which is `ghcr.io/tamnd/cpython-internals/cpython:release@sha256:fb55d6afcf053c974de6447fafbd2be6af20cdb9f596e25a0445607b8af981e3`. You do not need that build to read the numbers, and you do need it to produce them, which is why this is recorded rather than left as a cell you run. If you want to watch it happen yourself, `docker run --rm -i ghcr.io/tamnd/cpython-internals/cpython:release@sha256:fb55d6afcf053c974de6447fafbd2be6af20cdb9f596e25a0445607b8af981e3 python3 -` takes the program on standard input.\n", + "\n", + "Does a debug build behave the same way, and does freezing still pay for itself there?\n", + "\n", + "```python\n", + "\"\"\"What freezing the standard library into the binary is worth at startup.\n", + "\n", + "Import is written in Python, so it cannot be imported. CPython gets around that by compiling a\n", + "handful of modules during its own build and writing the bytecode into the binary as C arrays. The\n", + "first three are the import system itself, and they are what lets the interpreter get going at all.\n", + "\n", + "Everything after those three is a speed decision rather than a correctness one, and it can be\n", + "switched off with -X frozen_modules=off, so the cost of it can be measured rather than guessed.\n", + "Which way the switch sits by default is a build choice, so this program never relies on the\n", + "default. It asks for on and off explicitly and reports what the default happens to be.\n", + "\n", + "Two numbers matter. The first is how many code objects a bare startup reads off disk, which -v\n", + "prints one line for, so it is a count rather than a timing and it is the same on any machine. The\n", + "second is wall clock, measured with the two cases alternating, because anything that runs one case\n", + "forty times and then the other forty times is measuring the state of the page cache instead.\n", + "\"\"\"\n", + "\n", + "import _imp\n", + "import statistics\n", + "import subprocess\n", + "import sys\n", + "import time\n", + "\n", + "ROUNDS = 40\n", + "ON = [\"-X\", \"frozen_modules=on\"]\n", + "OFF = [\"-X\", \"frozen_modules=off\"]\n", + "ORIGIN = \"import os; print(os.__spec__.origin)\"\n", + "BOOTSTRAP = \"import sys; print(sys.modules['_frozen_importlib'].__spec__.origin)\"\n", + "COUNT = (\n", + " \"import sys\\n\"\n", + " \"specs = [getattr(m, '__spec__', None) for m in sys.modules.values()]\\n\"\n", + " \"print(len(sys.modules), sum(1 for s in specs if s is not None and s.origin == 'frozen'))\\n\"\n", + ")\n", + "\n", + "\n", + "def child(flags, args):\n", + " \"\"\"Run this same interpreter again with those flags and hand back the finished process.\"\"\"\n", + " return subprocess.run(\n", + " [sys.executable, *flags, *args], capture_output=True, text=True, check=True\n", + " )\n", + "\n", + "\n", + "def origin(flags):\n", + " \"\"\"Ask a fresh interpreter where the os module it just imported came from.\"\"\"\n", + " return child(flags, [\"-c\", ORIGIN]).stdout.strip()\n", + "\n", + "\n", + "def loaded(flags):\n", + " \"\"\"Ask a fresh interpreter how many modules it loaded and how many came out of the binary.\"\"\"\n", + " return [int(part) for part in child(flags, [\"-c\", COUNT]).stdout.split()]\n", + "\n", + "\n", + "def files_read(flags):\n", + " \"\"\"Count the code objects a fresh interpreter reads off disk, which -v prints one per line.\"\"\"\n", + " printed = child(flags, [\"-v\", \"-c\", \"pass\"]).stderr\n", + " return sum(1 for line in printed.splitlines() if line.startswith(\"# code object from\"))\n", + "\n", + "\n", + "def import_work(flags):\n", + " \"\"\"Add up the self time -X importtime reports, in microseconds.\"\"\"\n", + " printed = child([\"-X\", \"importtime\", *flags], [\"-c\", \"pass\"]).stderr\n", + " total = 0\n", + " for line in printed.splitlines():\n", + " if line.startswith(\"import time:\"):\n", + " first = line.removeprefix(\"import time:\").split(\"|\")[0].strip()\n", + " if first.isdigit():\n", + " total += int(first)\n", + " return total\n", + "\n", + "\n", + "def startup_ms(flags):\n", + " \"\"\"Wall clock milliseconds for one whole interpreter startup that does nothing at all.\"\"\"\n", + " started = time.perf_counter()\n", + " child(flags, [\"-c\", \"pass\"])\n", + " return (time.perf_counter() - started) * 1000\n", + "\n", + "\n", + "def alternating(measure, rounds):\n", + " \"\"\"Measure both cases once each per round, so neither one gets the cold cache every time.\"\"\"\n", + " got = {\"on\": [], \"off\": []}\n", + " for _ in range(rounds):\n", + " got[\"on\"].append(measure(ON))\n", + " got[\"off\"].append(measure(OFF))\n", + " return got\n", + "\n", + "\n", + "_imp._override_frozen_modules_for_tests(1)\n", + "names = _imp._frozen_module_names()\n", + "frozen_os = _imp.get_frozen_object(\"os\")\n", + "_imp._override_frozen_modules_for_tests(0)\n", + "\n", + "print(\"names compiled into this binary:\", len(names))\n", + "print(\"the three that cannot be turned off:\", \", \".join(names[:3]))\n", + "print(\"filename on the frozen code object for os:\", frozen_os.co_filename)\n", + "print(\"bytes of bytecode in it:\", len(frozen_os.co_code))\n", + "print(\"this build uses them unless told otherwise:\", origin([]) == \"frozen\")\n", + "print(\"where os comes from with the flag on:\", origin(ON))\n", + "print(\"where os comes from with the flag off:\", origin(OFF).rpartition(\"/\")[2])\n", + "print(\"_frozen_importlib with the flag off:\", child(OFF, [\"-c\", BOOTSTRAP]).stdout.strip())\n", + "\n", + "on_total, on_frozen = loaded(ON)\n", + "off_total, off_frozen = loaded(OFF)\n", + "print(\"modules a bare startup loads, flag on:\", on_total)\n", + "print(\"modules a bare startup loads, flag off:\", off_total)\n", + "print(\"of those, frozen with the flag on:\", on_frozen)\n", + "print(\"of those, frozen with the flag off:\", off_frozen)\n", + "print(\"code objects read off disk with the flag on:\", files_read(ON))\n", + "print(\"code objects read off disk with the flag off:\", files_read(OFF))\n", + "\n", + "work = alternating(import_work, 10)\n", + "print(f\"~ import work reported by importtime, frozen on: {min(work['on'])} us\")\n", + "print(f\"~ import work reported by importtime, frozen off: {min(work['off'])} us\")\n", + "\n", + "runs = alternating(startup_ms, ROUNDS)\n", + "fast_on, fast_off = min(runs[\"on\"]), min(runs[\"off\"])\n", + "print(f\"~ fastest startup with the flag on: {fast_on:.1f} ms\")\n", + "print(f\"~ fastest startup with the flag off: {fast_off:.1f} ms\")\n", + "print(f\"~ middle startup with the flag on: {statistics.median(runs['on']):.1f} ms\")\n", + "print(f\"~ middle startup with the flag off: {statistics.median(runs['off']):.1f} ms\")\n", + "share = (1 - fast_on / fast_off) * 100\n", + "print(f\"~ share of a startup that freezing gives back: {share:.1f} percent\")\n", + "```\n", + "\n", + "```text\n", + "names compiled into this binary: 33\n", + "the three that cannot be turned off: _frozen_importlib, _frozen_importlib_external, zipimport\n", + "filename on the frozen code object for os: \n", + "bytes of bytecode in it: 3360\n", + "this build uses them unless told otherwise: False\n", + "where os comes from with the flag on: frozen\n", + "where os comes from with the flag off: os.py\n", + "_frozen_importlib with the flag off: frozen\n", + "modules a bare startup loads, flag on: 33\n", + "modules a bare startup loads, flag off: 33\n", + "of those, frozen with the flag on: 17\n", + "of those, frozen with the flag off: 3\n", + "code objects read off disk with the flag on: 0\n", + "code objects read off disk with the flag off: 13\n", + "~ import work reported by importtime, frozen on: 20150 us\n", + "~ import work reported by importtime, frozen off: 22919 us\n", + "~ fastest startup with the flag on: 41.1 ms\n", + "~ fastest startup with the flag off: 45.6 ms\n", + "~ middle startup with the flag on: 52.3 ms\n", + "~ middle startup with the flag off: 59.0 ms\n", + "~ share of a startup that freezing gives back: 9.9 percent\n", + "```\n", + "\n", + "That ran on Python 3.15.0rc1 in the debug build this project publishes, which is `ghcr.io/tamnd/cpython-internals/cpython:debug@sha256:7baea8f3dd4de2e4c3b020543729b147e636494ae9758dabffb4675793e37170`. You do not need that build to read the numbers, and you do need it to produce them, which is why this is recorded rather than left as a cell you run. If you want to watch it happen yourself, `docker run --rm -i ghcr.io/tamnd/cpython-internals/cpython:debug@sha256:7baea8f3dd4de2e4c3b020543729b147e636494ae9758dabffb4675793e37170 python3 -` takes the program on standard input.\n", + "\n", + "![bars comparing startup milliseconds with frozen modules on and off, on a release build and a debug build](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r04-frozen-modules/diagrams/what-freezing-saves.svg)\n", + "\n", + "Both builds ship the same 33 frozen names, both read 13 files when told not to use them, and both save around a tenth of a startup by using them. What differs is the line that says whether the build uses them unless told otherwise: `True` on the release build and `False` on the debug one.\n", + "\n", + "That is a deliberate choice in the config defaults [Python/initconfig.c:1193-1201@v3.15.0rc1#use_frozen_modules](https://github.com/python/cpython/blob/v3.15.0rc1/Python/initconfig.c#L1193-L1201). A debug build is the build you use when you are editing `Lib/os.py`, and it would be miserable if your edits were ignored until you re ran the freeze step. So debug builds read the files, and everybody else gets the speed.\n", + "\n", + "A build configured with --with-pydebug leaves frozen modules off by default while a release build leaves them on, so the same interpreter version can disagree with itself about where os came from" + ] + }, + { + "cell_type": "markdown", + "id": "r04-32", + "metadata": {}, + "source": [ + "## Try it yourself\n", + "\n", + "Four things, in rough order of how much you will learn.\n", + "\n", + "Run `python -X frozen_modules=off -X importtime -c pass` and then the same command without the flag, and compare the two trees. Every line that appears only in the first run is a module the frozen build got for free.\n", + "\n", + "Point the disassembly cell at `_frozen_importlib_external` instead and then open the file at those lines. All eight imports are at the top, written completely normally, because by the time that module loads there is an import system to use.\n", + "\n", + "Read the last few lines of `Python/frozen.c` [Python/frozen.c:140-144@v3.15.0rc1#PyImport_FrozenModules](https://github.com/python/cpython/blob/v3.15.0rc1/Python/frozen.c#L140-L144). There is a fourth pointer there, it starts as `NULL`, and the comment invites an embedding application to point it at an array of its own. That is the supported way to ship a single file executable with a standard library inside it, and `look_up_frozen` checks it before the standard library group.\n", + "\n", + "Delete `stat` from `sys.modules`, turn the switch off, import it, and look at `stat.__spec__.cached`. A file import can write a `.pyc` and a frozen one has nothing to write, so the two runs leave different things behind on your disk.\n", + "\n", + "## What you now know\n", + "\n", + "The import system is written in Python and cannot be imported, so it is compiled during CPython's build and stored in the binary as a C array. The C startup code loads it directly and hands it `sys` and `_imp` as arguments.\n", + "\n", + "`importlib._bootstrap` has no imports in its module body at all. `importlib._bootstrap_external`, which loads second, has eight, because by then import works.\n", + "\n", + "A frozen module is a marshalled code object and nothing more. Loading one is a read out of memory plus an `exec`.\n", + "\n", + "There are 33 of them in a stock 3.15 build, in three groups. The three bootstrap names are searched no matter what. The nineteen standard library names and the eleven test names sit behind a flag.\n", + "\n", + "A frozen module still knows the path it was compiled from, because the loader parks it on the spec and copies it to `__file__`. That is why tracebacks say `` and still show you real source lines.\n", + "\n", + "`-X frozen_modules=off` is a supported way to see the difference, and `_imp._override_frozen_modules_for_tests` does the same thing without starting a new process.\n", + "\n", + "Freezing is worth about a tenth of a startup and thirteen files that never get opened. Debug builds leave it off by default so that editing the standard library works the way you expect.\n", + "\n", + "## What is next\n", + "\n", + "R05 is about the newest thing in this area. PEP 810 adds lazy imports to 3.15: an import that binds the name now and does the actual loading the first time somebody reads it.\n", + "\n", + "That changes the shape of everything R03 and R04 established. The module is not in `sys.modules` when the statement finishes, the finders have not been asked yet, and the cost has moved from startup to whenever the name is first touched. The disassembly cell in this lesson already hinted at it, because on 3.15 an `IMPORT_NAME` argument prints with an eager or lazy marker attached to it." + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/probes/pyodide/lessons.json b/probes/pyodide/lessons.json index 6685395..d53b9ec 100644 --- a/probes/pyodide/lessons.json +++ b/probes/pyodide/lessons.json @@ -329,7 +329,7 @@ { "name": "c05-18", "status": "ok", - "printed": " the collector is scheduled when generation zero passes 2000\n json.loads built them in 394 ms, collector ran 1 times\n a list display did it in 525 ms, collector ran 89 times\n the two of them built the same number of dicts: True\n" + "printed": " the collector is scheduled when generation zero passes 2000\n json.loads built them in 103 ms, collector ran 1 times\n a list display did it in 87 ms, collector ran 89 times\n the two of them built the same number of dicts: True\n" }, { "name": "c05-22", @@ -622,7 +622,7 @@ { "name": "e04-09", "status": "ok", - "printed": " a plain object at 25501824 mod 4 0 mod 8 0\n a list at 24495704 mod 4 0 mod 8 0\n a dict at 23079768 mod 4 0 mod 8 0\n a string at 24173920 mod 4 0 mod 8 0\n a big integer at 20337432 mod 4 0 mod 8 0\n a float at 19009064 mod 4 0 mod 8 0\n a module at 14795336 mod 4 0 mod 8 0\n None at 3200984 mod 4 0 mod 8 0\n the number 7 at 3253660 mod 4 0 mod 8 4\n" + "printed": " a plain object at 24789296 mod 4 0 mod 8 0\n a list at 22701120 mod 4 0 mod 8 0\n a dict at 18719368 mod 4 0 mod 8 0\n a string at 21687096 mod 4 0 mod 8 0\n a big integer at 20503800 mod 4 0 mod 8 0\n a float at 23572240 mod 4 0 mod 8 0\n a module at 14795336 mod 4 0 mod 8 0\n None at 3200984 mod 4 0 mod 8 0\n the number 7 at 3253660 mod 4 0 mod 8 4\n" }, { "name": "e04-12", @@ -691,12 +691,12 @@ { "name": "e05-18", "status": "ok", - "printed": " no_try 255.0 ns per iteration\n with_try 113.6 ns per iteration\n checking 482.4 ns per iteration\n" + "printed": " no_try 116.3 ns per iteration\n with_try 115.3 ns per iteration\n checking 250.0 ns per iteration\n" }, { "name": "e05-21", "status": "ok", - "printed": " through 1 frames 688.2 ns\n through 5 frames 5310.6 ns\n through 20 frames 15107.6 ns\n through 50 frames 39381.7 ns\n" + "printed": " through 1 frames 1614.7 ns\n through 5 frames 2396.0 ns\n through 20 frames 7985.1 ns\n through 50 frames 19053.1 ns\n" }, { "name": "e05-24", @@ -711,7 +711,7 @@ { "name": "e05-29", "status": "ok", - "printed": " for loop table 0 bytes 51.8 ns per item\n while and next table 12 bytes 209.1 ns per item\n" + "printed": " for loop table 0 bytes 73.4 ns per item\n while and next table 12 bytes 96.0 ns per item\n" }, { "name": "e05-32", @@ -780,7 +780,7 @@ { "name": "e06-31", "status": "ok", - "printed": " every value an int 42.6 ns per addition\n every value a float 44.2 ns per addition\n alternating between the two 146.0 ns per addition\n" + "printed": " every value an int 54.1 ns per addition\n every value a float 47.1 ns per addition\n alternating between the two 87.6 ns per addition\n" }, { "name": "e06-34", @@ -834,7 +834,7 @@ { "name": "e07-25", "status": "ok", - "printed": " with the jit on\n this build has no JIT in it, so there is nothing to switch on\n\n with the jit off, for comparison\n arithmetic loop 257.9 ns per iteration\n loop with a call 327.8 ns per iteration\n" + "printed": " with the jit on\n this build has no JIT in it, so there is nothing to switch on\n\n with the jit off, for comparison\n arithmetic loop 174.0 ns per iteration\n loop with a call 203.4 ns per iteration\n" } ] }, @@ -863,7 +863,7 @@ { "name": "e08-11", "status": "ok", - "printed": " nobody watching 202.1 ns per turn\n callback counts the line 1500.9 ns per turn, 2000015 calls\n callback returns DISABLE 95.2 ns per turn\n" + "printed": " nobody watching 109.2 ns per turn\n callback counts the line 1042.7 ns per turn, 2000015 calls\n callback returns DISABLE 195.6 ns per turn\n" }, { "name": "e08-14", @@ -1020,17 +1020,17 @@ { "name": "e11-15", "status": "ok", - "printed": " extra lines instructions run nanoseconds cost of one more\n 0 7010 57631 \n 5 17010 159905 10.23\n 10 27010 395546 23.56\n 20 47010 605512 10.50\n 40 87010 1123959 12.96\n" + "printed": " extra lines instructions run nanoseconds cost of one more\n 0 7010 57260 \n 5 17010 91379 3.41\n 10 27010 340889 24.95\n 20 47010 529148 9.41\n 40 87010 862085 8.32\n" }, { "name": "e11-18", "status": "ok", - "printed": " the same job instructions nanoseconds each\n a while loop 20012 295822 14.78\n a for loop 10008 118644 11.85\n a comprehension 6011 82331 13.70\n" + "printed": " the same job instructions nanoseconds each\n a while loop 20012 367055 18.34\n a for loop 10008 139560 13.94\n a comprehension 6011 112854 18.77\n" }, { "name": "e11-21", "status": "ok", - "printed": " one bytecode instruction 9.01 ns\n one call to an empty def 151.35 ns beyond the code that calls it\n so a call costs about 17 instructions\n" + "printed": " one bytecode instruction 9.41 ns\n one call to an empty def 46.61 ns beyond the code that calls it\n so a call costs about 5 instructions\n" } ] }, @@ -1489,7 +1489,7 @@ { "name": "f10-11", "status": "ok", - "printed": " -- MAKE_CELL 0 (a)\n MAKE_CELL 6 (n)\n\n 1 RESUME 0\n\n 2 LOAD_SMALL_INT 0\n STORE_DEREF 6 (n)\n\n 4 LOAD_FAST_BORROW 0 (a)\n LOAD_FAST_BORROW 6 (n)\n BUILD_TUPLE 2\n LOAD_CONST 1 ()\n MAKE_FUNCTION\n SET_FUNCTION_ATTRIBUTE 8 (closure)\n STORE_FAST 5 (inner)\n\n 7 LOAD_FAST_BORROW 5 (inner)\n RETURN_VALUE\n\nDisassembly of :\n -- COPY_FREE_VARS 2\n\n 4 RESUME 0\n\n 5 LOAD_DEREF 0 (a)\n LOAD_DEREF 1 (n)\n BINARY_OP 0 (+)\n RETURN_VALUE\n" + "printed": " -- MAKE_CELL 0 (a)\n MAKE_CELL 6 (n)\n\n 1 RESUME 0\n\n 2 LOAD_SMALL_INT 0\n STORE_DEREF 6 (n)\n\n 4 LOAD_FAST_BORROW 0 (a)\n LOAD_FAST_BORROW 6 (n)\n BUILD_TUPLE 2\n LOAD_CONST 1 ()\n MAKE_FUNCTION\n SET_FUNCTION_ATTRIBUTE 8 (closure)\n STORE_FAST 5 (inner)\n\n 7 LOAD_FAST_BORROW 5 (inner)\n RETURN_VALUE\n\nDisassembly of :\n -- COPY_FREE_VARS 2\n\n 4 RESUME 0\n\n 5 LOAD_DEREF 0 (a)\n LOAD_DEREF 1 (n)\n BINARY_OP 0 (+)\n RETURN_VALUE\n" }, { "name": "f10-13", @@ -1581,12 +1581,12 @@ { "name": "f12-07", "status": "ok", - "printed": " greet.py is 54 bytes of text\n greet.cpython-314.pyc is 251 bytes\n\n header 2b 0e 0d 0a 00 00 00 00 5f 71 9c 6a 36 00 00 00\n the rest 235 bytes, and marshal.loads gives back a\n code object\n" + "printed": " greet.py is 54 bytes of text\n greet.cpython-314.pyc is 251 bytes\n\n header 2b 0e 0d 0a 00 00 00 00 63 a0 9c 6a 36 00 00 00\n the rest 235 bytes, and marshal.loads gives back a\n code object\n" }, { "name": "f12-10", "status": "ok", - "printed": " magic 168627755\n flags 0\n mtime 1788637535 source 1788637535\n size 54 source 54\n" + "printed": " magic 168627755\n flags 0\n mtime 1788649571 source 1788649571\n size 54 source 54\n" }, { "name": "f12-13", @@ -1620,7 +1620,7 @@ { "name": "f12-29", "status": "ok", - "printed": " one bit flipped in the magic: bad magic number in 'broken': b'*\\x0e\\r\\n'\n\n put a wrong mtime in greet.cpython-314.pyc: 1788637600\n imported it with no complaint: 'hi world'\n and the header now says 1788637600\n the file was quietly rewritten: False\n" + "printed": " one bit flipped in the magic: bad magic number in 'broken': b'*\\x0e\\r\\n'\n\n put a wrong mtime in greet.cpython-314.pyc: 1788649628\n imported it with no complaint: 'hi world'\n and the header now says 1788649628\n the file was quietly rewritten: False\n" } ] }, @@ -1963,7 +1963,7 @@ { "name": "m07-29", "status": "ok", - "printed": " frozen objects before 0\n tracked in generation 2 24945\n frozen after gc.freeze() 24945\n left in generation 2 0\n frozen after gc.unfreeze() 0\n" + "printed": " frozen objects before 0\n tracked in generation 2 24949\n frozen after gc.freeze() 24949\n left in generation 2 0\n frozen after gc.unfreeze() 0\n" } ] }, @@ -1987,7 +1987,7 @@ { "name": "m08-09", "status": "ok", - "printed": " objects in each generation [3, 0, 24931]\n as soon as it exists generations [0]\n after a pass over gen 0 generations [2]\n after a pass over gen 1 generations [2]\n" + "printed": " objects in each generation [3, 0, 24935]\n as soon as it exists generations [0]\n after a pass over gen 0 generations [2]\n after a pass over gen 1 generations [2]\n" }, { "name": "m08-12", @@ -2002,12 +2002,12 @@ { "name": "m08-17", "status": "ok", - "printed": " cycles on the heap 200000\n one full pass over it 37 ms\n and over an empty one 7 ms\n" + "printed": " cycles on the heap 200000\n one full pass over it 66 ms\n and over an empty one 24 ms\n" }, { "name": "m08-20", "status": "ok", - "printed": " this version does not report candidates, so there is nothing to compare\n objects the collector is tracking 24947\n" + "printed": " this version does not report candidates, so there is nothing to compare\n objects the collector is tracking 24951\n" } ] }, @@ -2085,7 +2085,7 @@ { "name": "o01-07", "status": "ok", - "printed": " a pointer here is 4 bytes, so the header is 8\n it lives at 21891664\n\n first word, read as a count 1\n sys.getrefcount, less its own argument 1\n the same number: True\n\n second word 20520272\n id(Thing) is 20520272\n so the second word is the type pointer: True\n\n after a second name, the count reads 2\n after deleting that name, it reads 1\n" + "printed": " a pointer here is 4 bytes, so the header is 8\n it lives at 23966208\n\n first word, read as a count 1\n sys.getrefcount, less its own argument 1\n the same number: True\n\n second word 19430888\n id(Thing) is 19430888\n so the second word is the type pointer: True\n\n after a second name, the count reads 2\n after deleting that name, it reads 1\n" }, { "name": "o01-10", @@ -2958,7 +2958,61 @@ { "name": "r03-22", "status": "ok", - "printed": " first time, source file, no .pyc yet: 403.1 microseconds\n from a file, with a .pyc already there: 264.6 microseconds\n asking again for one already loaded: 273.4 nanoseconds\n the second of those is 968 times the third\n" + "printed": " first time, source file, no .pyc yet: 393.0 microseconds\n from a file, with a .pyc already there: 198.7 microseconds\n asking again for one already loaded: 302.2 nanoseconds\n the second of those is 657 times the third\n" + } + ] + }, + { + "slug": "r04-frozen-modules", + "cells": [ + { + "name": "r04-03", + "status": "ok" + }, + { + "name": "r04-05", + "status": "ok", + "printed": "cpython 3.14.2 on Emscripten wasm32, WebAssembly\n note: this is 3.14, and everything here is written against 3.15, so bytecode and some object layouts will differ from the prose\n" + }, + { + "name": "r04-07", + "status": "ok", + "printed": " _frozen_importlib 0 IMPORT_NAME in the module body\n _frozen_importlib_external 8 IMPORT_NAME in the module body\n os 18 IMPORT_NAME in the module body\n site 7 IMPORT_NAME in the module body\n\n the first modules any process has: ['sys', 'builtins', '_frozen_importlib', '_imp', '_thread', '_warnings']\n" + }, + { + "name": "r04-10", + "status": "ok", + "printed": " names frozen into this binary: 28\n\n the import system: _frozen_importlib, _frozen_importlib_external, zipimport\n the standard library: abc, codecs, io, _collections_abc, _sitebuiltins, genericpath, ntpath, posixpath, os, site, stat, importlib.util, importlib.machinery, runpy\n hello world, for the test suite: 11 names\n\n all of that bytecode, marshalled: 438,584 bytes\n" + }, + { + "name": "r04-13", + "status": "ok", + "printed": " what get_frozen_object gives back: code\n its co_filename: \n its co_name: \n bytes of bytecode in it: 3,088\n\n is_frozen('os'): True\n frozen names that are packages: __phello_alias__, __phello__, __phello__.ham\n\n find_frozen('os' ) -> package False, frozen from 'os'\n find_frozen('__phello__.ham' ) -> package True, frozen from '__phello__.ham'\n find_frozen('_frozen_importlib' ) -> package False, frozen from 'importlib._bootstrap'\n" + }, + { + "name": "r04-16", + "status": "ok", + "printed": " os.__spec__.origin: frozen\n os.__spec__.cached: None\n os.__spec__.has_location: False\n os.__spec__.loader_state: os.py\n os.__file__ is a file that exists: False\n\n the code object for os.get_exec_path says it came from: \n linecache asked for that line with nothing else:\n ''\n and asked again with the module's globals:\n ''\n" + }, + { + "name": "r04-19", + "status": "ok", + "printed": " frozen names on: 28\n FrozenImporter.find_spec('stat'): ModuleSpec(name='stat', loader=, origin='frozen')\n\n frozen names off: ['_frozen_importlib', '_frozen_importlib_external', 'zipimport']\n FrozenImporter.find_spec('stat'): None\n and the bootstrap is still there: frozen\n is_frozen('os') now says: False\n\n back on again, frozen names: 28\n" + }, + { + "name": "r04-22", + "status": "ok", + "printed": " with the switch off, stat came from: stat.py\n its loader: zipimporter\n with the switch on, stat came from: frozen\n its loader: FrozenImporter\n\n same constant either way: True\n" + }, + { + "name": "r04-25", + "status": "ok", + "printed": " asking FrozenImporter for os: 3.31 microseconds\n asking PathFinder for the same name: 13311.16 microseconds\n this runtime keeps no pyc for os, so the last two rows have nothing to measure\n" + }, + { + "name": "r04-29", + "status": "ok", + "printed": " this runtime cannot start a second process, so this cell has nothing to show\n" } ] }, @@ -3344,7 +3398,7 @@ { "name": "t04-30", "status": "ok", - "printed": "free variables: ('total',)\nthe cells: (,)\nfirst call: 1\nsecond call: 2\nthe cell now: 2\n" + "printed": "free variables: ('total',)\nthe cells: (,)\nfirst call: 1\nsecond call: 2\nthe cell now: 2\n" }, { "name": "t04-33", @@ -3621,7 +3675,7 @@ { "name": "t07-38", "status": "ok", - "printed": "asked twice, got the same object: True\nand it is still here after the call returned: \nits name: make_one\n" + "printed": "asked twice, got the same object: True\nand it is still here after the call returned: \nits name: make_one\n" }, { "name": "t07-40", @@ -3655,12 +3709,12 @@ { "name": "t08-11", "status": "ok", - "printed": "NoneType at 0x30d7d8, refcount is parked, this object is never freed, 8 bytes, not tracked\nint at 0x31a7cc, refcount is parked, this object is never freed, 16 bytes, not tracked\nstr at 0x14f8948, 3 reference(s), 26 bytes, not tracked\nlist at 0x16f2bd8, 2 reference(s), 44 bytes, tracked by the cycle collector\ndict at 0x178a0f0, 2 reference(s), 108 bytes, tracked by the cycle collector\nfunction at 0x1655238, 4 reference(s), 84 bytes, tracked by the cycle collector\n" + "printed": "NoneType at 0x30d7d8, refcount is parked, this object is never freed, 8 bytes, not tracked\nint at 0x31a7cc, refcount is parked, this object is never freed, 16 bytes, not tracked\nstr at 0x1680ac8, 3 reference(s), 26 bytes, not tracked\nlist at 0x15806d0, 2 reference(s), 44 bytes, tracked by the cycle collector\ndict at 0x17116f8, 2 reference(s), 108 bytes, tracked by the cycle collector\nfunction at 0x129db48, 4 reference(s), 84 bytes, tracked by the cycle collector\n" }, { "name": "t08-14", "status": "ok", - "printed": "a == b True same contents\na is b False different objects\na is c True same object\n\nid(a) 0x116cbe8\nid(b) 0x16f2ea8\nid(c) 0x116cbe8\n" + "printed": "a == b True same contents\na is b False different objects\na is c True same object\n\nid(a) 0x16025c0\nid(b) 0x17dc300\nid(c) 0x16025c0\n" }, { "name": "t08-17", @@ -3700,7 +3754,7 @@ { "name": "t08-35", "status": "ok", - "printed": "dict at 0x11ef2a0\ndict at 0x16c2fe0\nlist at 0x16f2d90\n" + "printed": "dict at 0x13eb218\ndict at 0x1767720\nlist at 0x181e0f0\n" }, { "name": "t08-37", @@ -3759,7 +3813,7 @@ { "name": "t09-21", "status": "ok", - "printed": "Node at 0x17fa588 -> Node at 0x18beb30 -> Node at 0x1600e78 -> Node at 0x17fa588\n" + "printed": "Node at 0x18355a0 -> Node at 0x14bdf08 -> Node at 0x17d45c0 -> Node at 0x18355a0\n" }, { "name": "t09-23", @@ -3789,7 +3843,7 @@ { "name": "t09-37", "status": "ok", - "printed": "first object was at 0x14b5698\nsecond object is at 0x14b5698\nsame address reused -> True\n" + "printed": "first object was at 0x132d928\nsecond object is at 0x132d928\nsame address reused -> True\n" } ] }, diff --git a/probes/pyodide/lessons.md b/probes/pyodide/lessons.md index 79ab90b..5d8cd36 100644 --- a/probes/pyodide/lessons.md +++ b/probes/pyodide/lessons.md @@ -2,7 +2,7 @@ Generated by `just build-probe`. Do not edit by hand, the change will be overwritten. -74 lesson(s) on Pyodide 3.14.2: 74 ran end to end, 723 cell(s) in total. +75 lesson(s) on Pyodide 3.14.2: 75 ran end to end, 733 cell(s) in total. The checks in `report.md` next to this ask whether a surface exists. This runs the lessons themselves: every code cell of every notebook, in order, in one Pyodide runtime, with `pyxray` mounted off the disk rather than installed. The install cell is the one thing changed, and only its `%pip` line, which a reader in a browser does not need either. @@ -70,6 +70,7 @@ The checks in `report.md` next to this ask whether a surface exists. This runs t | r01-before-your-first-line | 8 | runs end to end | | r02-where-the-state-lives | 8 | runs end to end | | r03-what-import-does | 10 | runs end to end | +| r04-frozen-modules | 10 | runs end to end | | t01-one-line-seven-stages | 18 | runs end to end | | t02-text-becomes-tokens | 32 | runs end to end | | t03-tokens-become-a-tree | 13 | runs end to end | diff --git a/pyxray/src/pyxray/glossary.py b/pyxray/src/pyxray/glossary.py index c94e4e9..d380524 100644 --- a/pyxray/src/pyxray/glossary.py +++ b/pyxray/src/pyxray/glossary.py @@ -1839,6 +1839,42 @@ def anchor(name: str) -> str: see=("meta path finder", "interpreter state"), met="R03", ), + Term( + name="frozen module", + short="A module whose bytecode was compiled during CPython's own build and written into the binary.", + long="The build step compiles the source, marshals the code object and writes the bytes out as a C array in a generated header, so loading one is a read out of memory that was already mapped and no file is opened at all. Thirty three names are frozen in a stock 3.15 build. Three of them are the import system and cannot be switched off, and the rest are a startup speed decision that `-X frozen_modules=off` undoes.", + cite="Python/frozen.c:74-79@v3.15.0rc1#bootstrap_modules", + also=("`_imp.is_frozen`", "`-X frozen_modules`", "`Python/frozen.c`"), + see=("import bootstrap", "loader state", "marshal"), + met="R04", + ), + Term( + name="import bootstrap", + short="The C code that loads the Python import system before there is an import system.", + long="`init_importlib` imports `_frozen_importlib` straight out of the frozen table, builds the `_imp` module by hand, and then calls `_install` on the module it has just loaded, handing `sys` and `_imp` in as arguments. That is why `Lib/importlib/_bootstrap.py` never writes `import sys` anywhere. The two modules it needs are passed to it and assigned to globals in `_setup`.", + cite="Python/import.c:3389-3428@v3.15.0rc1#init_importlib", + also=("`init_importlib`", "`_setup`", "`_install`"), + see=("frozen module", "meta path finder"), + met="R04", + ), + Term( + name="loader state", + short="A free field on a module spec that a loader can use to remember whatever it likes.", + long="`ModuleSpec` sets `loader_state` to `None` and never looks at it again, so the field belongs to whichever loader filled the spec in. `FrozenImporter` puts a small namespace there holding the path the module was compiled from and the name it was frozen under, which is how `os.__file__` ends up pointing at a file that the import itself never opened.", + cite="Lib/importlib/_bootstrap.py:1106-1133@v3.15.0rc1#find_spec", + also=("`spec.loader_state`", "`origname`", "`_fix_up_module`"), + see=("module spec", "frozen module"), + met="R04", + ), + Term( + name="module alias", + short="A frozen entry that carries the name its source file really had.", + long="`importlib._bootstrap` is frozen under the name `_frozen_importlib`, because the interpreter needs it long before the `importlib` package exists as anything you could import. A table in `Python/frozen.c` maps one name to the other, and `_imp.find_frozen` hands the original back, so the loader can fix the module up afterwards and both names end up meaning the same module object.", + cite="Python/frozen.c:127-137@v3.15.0rc1#aliases", + also=("`aliases`", "`is_alias`", "`__origname__`"), + see=("frozen module", "import bootstrap"), + met="R04", + ), ), ) diff --git a/tools/tier1/src/tier1/experiments.py b/tools/tier1/src/tier1/experiments.py index 499190f..331779f 100644 --- a/tools/tier1/src/tier1/experiments.py +++ b/tools/tier1/src/tier1/experiments.py @@ -2606,6 +2606,158 @@ def in_threads(names): ) +PROGRAM_TWENTYTHREE = r'''"""What freezing the standard library into the binary is worth at startup. + +Import is written in Python, so it cannot be imported. CPython gets around that by compiling a +handful of modules during its own build and writing the bytecode into the binary as C arrays. The +first three are the import system itself, and they are what lets the interpreter get going at all. + +Everything after those three is a speed decision rather than a correctness one, and it can be +switched off with -X frozen_modules=off, so the cost of it can be measured rather than guessed. +Which way the switch sits by default is a build choice, so this program never relies on the +default. It asks for on and off explicitly and reports what the default happens to be. + +Two numbers matter. The first is how many code objects a bare startup reads off disk, which -v +prints one line for, so it is a count rather than a timing and it is the same on any machine. The +second is wall clock, measured with the two cases alternating, because anything that runs one case +forty times and then the other forty times is measuring the state of the page cache instead. +""" + +import _imp +import statistics +import subprocess +import sys +import time + +ROUNDS = 40 +ON = ["-X", "frozen_modules=on"] +OFF = ["-X", "frozen_modules=off"] +ORIGIN = "import os; print(os.__spec__.origin)" +BOOTSTRAP = "import sys; print(sys.modules['_frozen_importlib'].__spec__.origin)" +COUNT = ( + "import sys\n" + "specs = [getattr(m, '__spec__', None) for m in sys.modules.values()]\n" + "print(len(sys.modules), sum(1 for s in specs if s is not None and s.origin == 'frozen'))\n" +) + + +def child(flags, args): + """Run this same interpreter again with those flags and hand back the finished process.""" + return subprocess.run( + [sys.executable, *flags, *args], capture_output=True, text=True, check=True + ) + + +def origin(flags): + """Ask a fresh interpreter where the os module it just imported came from.""" + return child(flags, ["-c", ORIGIN]).stdout.strip() + + +def loaded(flags): + """Ask a fresh interpreter how many modules it loaded and how many came out of the binary.""" + return [int(part) for part in child(flags, ["-c", COUNT]).stdout.split()] + + +def files_read(flags): + """Count the code objects a fresh interpreter reads off disk, which -v prints one per line.""" + printed = child(flags, ["-v", "-c", "pass"]).stderr + return sum(1 for line in printed.splitlines() if line.startswith("# code object from")) + + +def import_work(flags): + """Add up the self time -X importtime reports, in microseconds.""" + printed = child(["-X", "importtime", *flags], ["-c", "pass"]).stderr + total = 0 + for line in printed.splitlines(): + if line.startswith("import time:"): + first = line.removeprefix("import time:").split("|")[0].strip() + if first.isdigit(): + total += int(first) + return total + + +def startup_ms(flags): + """Wall clock milliseconds for one whole interpreter startup that does nothing at all.""" + started = time.perf_counter() + child(flags, ["-c", "pass"]) + return (time.perf_counter() - started) * 1000 + + +def alternating(measure, rounds): + """Measure both cases once each per round, so neither one gets the cold cache every time.""" + got = {"on": [], "off": []} + for _ in range(rounds): + got["on"].append(measure(ON)) + got["off"].append(measure(OFF)) + return got + + +_imp._override_frozen_modules_for_tests(1) +names = _imp._frozen_module_names() +frozen_os = _imp.get_frozen_object("os") +_imp._override_frozen_modules_for_tests(0) + +print("names compiled into this binary:", len(names)) +print("the three that cannot be turned off:", ", ".join(names[:3])) +print("filename on the frozen code object for os:", frozen_os.co_filename) +print("bytes of bytecode in it:", len(frozen_os.co_code)) +print("this build uses them unless told otherwise:", origin([]) == "frozen") +print("where os comes from with the flag on:", origin(ON)) +print("where os comes from with the flag off:", origin(OFF).rpartition("/")[2]) +print("_frozen_importlib with the flag off:", child(OFF, ["-c", BOOTSTRAP]).stdout.strip()) + +on_total, on_frozen = loaded(ON) +off_total, off_frozen = loaded(OFF) +print("modules a bare startup loads, flag on:", on_total) +print("modules a bare startup loads, flag off:", off_total) +print("of those, frozen with the flag on:", on_frozen) +print("of those, frozen with the flag off:", off_frozen) +print("code objects read off disk with the flag on:", files_read(ON)) +print("code objects read off disk with the flag off:", files_read(OFF)) + +work = alternating(import_work, 10) +print(f"~ import work reported by importtime, frozen on: {min(work['on'])} us") +print(f"~ import work reported by importtime, frozen off: {min(work['off'])} us") + +runs = alternating(startup_ms, ROUNDS) +fast_on, fast_off = min(runs["on"]), min(runs["off"]) +print(f"~ fastest startup with the flag on: {fast_on:.1f} ms") +print(f"~ fastest startup with the flag off: {fast_off:.1f} ms") +print(f"~ middle startup with the flag on: {statistics.median(runs['on']):.1f} ms") +print(f"~ middle startup with the flag off: {statistics.median(runs['off']):.1f} ms") +share = (1 - fast_on / fast_off) * 100 +print(f"~ share of a startup that freezing gives back: {share:.1f} percent") +''' + + +WHAT_FREEZING_SAVES_AT_STARTUP = Experiment( + slug="r04-what-freezing-saves-at-startup", + lesson="R04", + title="A startup with the frozen standard library, and the same startup without it", + asks="What does compiling the standard library into the binary actually save at startup?", + needs=( + "it wants a machine that is not busy with anything else, because half of what it reports " + "is wall clock for a process that only lives for a few milliseconds" + ), + build="release", + program=PROGRAM_TWENTYTHREE, +) + + +WHAT_FREEZING_SAVES_ON_A_DEBUG_BUILD = Experiment( + slug="r04-what-freezing-saves-on-a-debug-build", + lesson="R04", + title="The same two startups on a build that does not use the frozen copies", + asks="Does a debug build behave the same way, and does freezing still pay for itself there?", + needs=( + "it needs a build configured with --with-pydebug, which is the one build that leaves the " + "frozen copies switched off by default, so the same program reports a different default" + ), + build="debug", + program=PROGRAM_TWENTYTHREE, +) + + EXPERIMENTS: tuple[Experiment, ...] = ( COMPILING_COSTS_NOTHING_THAT_LASTS, A_LEAK_YOU_CAN_SEE, @@ -2639,6 +2791,8 @@ def in_threads(names): WHAT_A_SECOND_INTERPRETER_COSTS_WITHOUT_THE_LOCK, HOW_MUCH_OF_AN_IMPORT_IS_PARALLEL, HOW_MUCH_OF_AN_IMPORT_IS_PARALLEL_WITHOUT_THE_LOCK, + WHAT_FREEZING_SAVES_AT_STARTUP, + WHAT_FREEZING_SAVES_ON_A_DEBUG_BUILD, ) From 2c4a77f94b1783b91e473b565f814f4884940788 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Sun, 6 Sep 2026 06:46:42 +0700 Subject: [PATCH 2/2] Read the -v listing without assuming the path is quoted CPython prints # code object from with a repr when it loads a cached pyc and without one when it compiles the source instead, and the cell only handled the first spelling, so it raised IndexError on a checkout with no cached bytecode next to the standard library. Take the marker off the front and strip the quotes if there are any, work the package name out from either an __init__.py or a __pycache__ directory, and print the suffix in the listing so the difference is visible rather than assumed. The claim now says every file holds a frozen name, with the pyc part conditional on the install. --- lessons/CLAIMS.md | 2 +- lessons/r04-frozen-modules/build.py | 19 ++++++---- lessons/r04-frozen-modules/r04.ipynb | 20 ++++++----- probes/pyodide/lessons.json | 52 ++++++++++++++-------------- 4 files changed, 51 insertions(+), 42 deletions(-) diff --git a/lessons/CLAIMS.md b/lessons/CLAIMS.md index fcae060..b56f324 100644 --- a/lessons/CLAIMS.md +++ b/lessons/CLAIMS.md @@ -761,7 +761,7 @@ Without it the exception becomes the rule and this goes back to being a book. | Turning frozen modules off in process removes the standard library and test names from the frozen table and leaves exactly the three bootstrap names, which no setting can remove | [`r04-19`](r04-frozen-modules/r04.ipynb) | | The same import statement produces a module with an origin of frozen or an origin of a file path depending only on the switch, and the module works identically either way | [`r04-22`](r04-frozen-modules/r04.ipynb) | | Loading a module from a file and loading it out of the binary end in the same unmarshal of the same bytes, and what freezing removes is the finder search and the file read in front of that | [`r04-25`](r04-frozen-modules/r04.ipynb) | -| Every file a bare startup reads only when frozen modules are switched off is a pyc for a module in the frozen standard library group, which means freezing saves file reading rather than compiling | [`r04-29`](r04-frozen-modules/r04.ipynb) | +| Every file a bare startup reads only when frozen modules are switched off holds a module that is in the frozen standard library group, and on an install with cached bytecode every one of them is an already compiled pyc | [`r04-29`](r04-frozen-modules/r04.ipynb) | | A build configured with --with-pydebug leaves frozen modules off by default while a release build leaves them on, so the same interpreter version can disagree with itself about where os came from | not observable from Python: it needs two builds of the same source in two containers, and one notebook is only ever one of them | ## T01. One line, seven stages diff --git a/lessons/r04-frozen-modules/build.py b/lessons/r04-frozen-modules/build.py index 919eed2..abe2a3d 100644 --- a/lessons/r04-frozen-modules/build.py +++ b/lessons/r04-frozen-modules/build.py @@ -407,7 +407,7 @@ def per_call(work, rounds=2000): The list is the part that travels. Timings depend on your disk and your processor, but which files a startup opens does not, and `-v` prints one line for each code object it reads. -{lesson.claim("Every file a bare startup reads only when frozen modules are switched off is a pyc for a module in the frozen standard library group, which means freezing saves file reading rather than compiling")} +{lesson.claim("Every file a bare startup reads only when frozen modules are switched off holds a module that is in the frozen standard library group, and on an install with cached bytecode every one of them is an already compiled pyc")} """) @@ -424,15 +424,18 @@ def files_read(flags): started = subprocess.run( [sys.executable, *flags, "-v", "-c", "pass"], capture_output=True, text=True, check=True ) + marker = "# code object from " lines = started.stderr.splitlines() - return [line.split("'")[1] for line in lines if line.startswith("# code object from")] + return [line.removeprefix(marker).strip("'") for line in lines if line.startswith(marker)] def module_name(path): - \"\"\"Turn the path of a pyc file back into the name of the module it holds.\"\"\" + \"\"\"Turn the path of a file back into the name of the module whose code it holds.\"\"\" parts = pathlib.Path(path).parts stem = parts[-1].split(".")[0] - return parts[-3] if stem == "__init__" else stem + if stem != "__init__": + return stem + return parts[-3] if parts[-2] == "__pycache__" else parts[-2] try: @@ -443,20 +446,22 @@ def module_name(path): tails = {name.rpartition(".")[2] for name in names} for path in [one for one in off if one not in on]: held = module_name(path) - print(f" {held:20} is in the frozen list: {held in tails}") + kind = pathlib.Path(path).suffix.lstrip(".") + print(f" {held:20} a {kind:3} for a frozen name: {held in tails}") except OSError: print(" this runtime cannot start a second process, so this cell has nothing to show") """, varies=( "the totals depend on what your interpreter loads at startup, and a virtual environment " "adds a handful of files to both runs, but the list underneath is the frozen standard " - "library either way, minus whichever of those names your version had not frozen yet" + "library either way, minus whichever of those names your version had not frozen yet. An " + "install with no cached bytecode next to the source reads py files there rather than pyc" ), ) lesson.md(f""" -Every name on that list is a `.pyc` that had already been compiled, so what freezing saves at startup is not compiling anything. It is a dozen rounds of asking finders, listing directories, opening files and checking timestamps. +Every name on that list is in the frozen standard library group, which is the point of the cell. The middle column is worth a look too. Where it says `pyc`, that file had already been compiled, so what freezing saved there was not compilation at all, it was a round of asking finders, listing directories, opening a file and checking a timestamp. Where it says `py`, which happens on an install that ships no cached bytecode, the flag off run pays for the compile as well. Now the clock. Two Tier 1 recordings run the same program in a container, once on a release build and once on a debug build, and the interesting part is that the two builds disagree about the default. diff --git a/lessons/r04-frozen-modules/r04.ipynb b/lessons/r04-frozen-modules/r04.ipynb index 1cb09b4..53972c7 100644 --- a/lessons/r04-frozen-modules/r04.ipynb +++ b/lessons/r04-frozen-modules/r04.ipynb @@ -494,7 +494,7 @@ "\n", "The list is the part that travels. Timings depend on your disk and your processor, but which files a startup opens does not, and `-v` prints one line for each code object it reads.\n", "\n", - "Every file a bare startup reads only when frozen modules are switched off is a pyc for a module in the frozen standard library group, which means freezing saves file reading rather than compiling" + "Every file a bare startup reads only when frozen modules are switched off holds a module that is in the frozen standard library group, and on an install with cached bytecode every one of them is an already compiled pyc" ] }, { @@ -503,7 +503,7 @@ "id": "r04-29", "metadata": { "cpython_internals": { - "varies": "the totals depend on what your interpreter loads at startup, and a virtual environment adds a handful of files to both runs, but the list underneath is the frozen standard library either way, minus whichever of those names your version had not frozen yet" + "varies": "the totals depend on what your interpreter loads at startup, and a virtual environment adds a handful of files to both runs, but the list underneath is the frozen standard library either way, minus whichever of those names your version had not frozen yet. An install with no cached bytecode next to the source reads py files there rather than pyc" } }, "outputs": [], @@ -519,15 +519,18 @@ " started = subprocess.run(\n", " [sys.executable, *flags, \"-v\", \"-c\", \"pass\"], capture_output=True, text=True, check=True\n", " )\n", + " marker = \"# code object from \"\n", " lines = started.stderr.splitlines()\n", - " return [line.split(\"'\")[1] for line in lines if line.startswith(\"# code object from\")]\n", + " return [line.removeprefix(marker).strip(\"'\") for line in lines if line.startswith(marker)]\n", "\n", "\n", "def module_name(path):\n", - " \"\"\"Turn the path of a pyc file back into the name of the module it holds.\"\"\"\n", + " \"\"\"Turn the path of a file back into the name of the module whose code it holds.\"\"\"\n", " parts = pathlib.Path(path).parts\n", " stem = parts[-1].split(\".\")[0]\n", - " return parts[-3] if stem == \"__init__\" else stem\n", + " if stem != \"__init__\":\n", + " return stem\n", + " return parts[-3] if parts[-2] == \"__pycache__\" else parts[-2]\n", "\n", "\n", "try:\n", @@ -538,7 +541,8 @@ " tails = {name.rpartition(\".\")[2] for name in names}\n", " for path in [one for one in off if one not in on]:\n", " held = module_name(path)\n", - " print(f\" {held:20} is in the frozen list: {held in tails}\")\n", + " kind = pathlib.Path(path).suffix.lstrip(\".\")\n", + " print(f\" {held:20} a {kind:3} for a frozen name: {held in tails}\")\n", "except OSError:\n", " print(\" this runtime cannot start a second process, so this cell has nothing to show\")" ] @@ -548,7 +552,7 @@ "id": "r04-30", "metadata": {}, "source": [ - "> **Version note.** the totals depend on what your interpreter loads at startup, and a virtual environment adds a handful of files to both runs, but the list underneath is the frozen standard library either way, minus whichever of those names your version had not frozen yet" + "> **Version note.** the totals depend on what your interpreter loads at startup, and a virtual environment adds a handful of files to both runs, but the list underneath is the frozen standard library either way, minus whichever of those names your version had not frozen yet. An install with no cached bytecode next to the source reads py files there rather than pyc" ] }, { @@ -556,7 +560,7 @@ "id": "r04-31", "metadata": {}, "source": [ - "Every name on that list is a `.pyc` that had already been compiled, so what freezing saves at startup is not compiling anything. It is a dozen rounds of asking finders, listing directories, opening files and checking timestamps.\n", + "Every name on that list is in the frozen standard library group, which is the point of the cell. The middle column is worth a look too. Where it says `pyc`, that file had already been compiled, so what freezing saved there was not compilation at all, it was a round of asking finders, listing directories, opening a file and checking a timestamp. Where it says `py`, which happens on an install that ships no cached bytecode, the flag off run pays for the compile as well.\n", "\n", "Now the clock. Two Tier 1 recordings run the same program in a container, once on a release build and once on a debug build, and the interesting part is that the two builds disagree about the default.\n", "\n", diff --git a/probes/pyodide/lessons.json b/probes/pyodide/lessons.json index d53b9ec..eeed9f3 100644 --- a/probes/pyodide/lessons.json +++ b/probes/pyodide/lessons.json @@ -329,7 +329,7 @@ { "name": "c05-18", "status": "ok", - "printed": " the collector is scheduled when generation zero passes 2000\n json.loads built them in 103 ms, collector ran 1 times\n a list display did it in 87 ms, collector ran 89 times\n the two of them built the same number of dicts: True\n" + "printed": " the collector is scheduled when generation zero passes 2000\n json.loads built them in 71 ms, collector ran 1 times\n a list display did it in 58 ms, collector ran 89 times\n the two of them built the same number of dicts: True\n" }, { "name": "c05-22", @@ -622,7 +622,7 @@ { "name": "e04-09", "status": "ok", - "printed": " a plain object at 24789296 mod 4 0 mod 8 0\n a list at 22701120 mod 4 0 mod 8 0\n a dict at 18719368 mod 4 0 mod 8 0\n a string at 21687096 mod 4 0 mod 8 0\n a big integer at 20503800 mod 4 0 mod 8 0\n a float at 23572240 mod 4 0 mod 8 0\n a module at 14795336 mod 4 0 mod 8 0\n None at 3200984 mod 4 0 mod 8 0\n the number 7 at 3253660 mod 4 0 mod 8 4\n" + "printed": " a plain object at 24125192 mod 4 0 mod 8 0\n a list at 25992152 mod 4 0 mod 8 0\n a dict at 21054416 mod 4 0 mod 8 0\n a string at 24297088 mod 4 0 mod 8 0\n a big integer at 25047000 mod 4 0 mod 8 0\n a float at 18916888 mod 4 0 mod 8 0\n a module at 14795336 mod 4 0 mod 8 0\n None at 3200984 mod 4 0 mod 8 0\n the number 7 at 3253660 mod 4 0 mod 8 4\n" }, { "name": "e04-12", @@ -691,12 +691,12 @@ { "name": "e05-18", "status": "ok", - "printed": " no_try 116.3 ns per iteration\n with_try 115.3 ns per iteration\n checking 250.0 ns per iteration\n" + "printed": " no_try 116.0 ns per iteration\n with_try 103.0 ns per iteration\n checking 224.4 ns per iteration\n" }, { "name": "e05-21", "status": "ok", - "printed": " through 1 frames 1614.7 ns\n through 5 frames 2396.0 ns\n through 20 frames 7985.1 ns\n through 50 frames 19053.1 ns\n" + "printed": " through 1 frames 1062.0 ns\n through 5 frames 2140.2 ns\n through 20 frames 6597.7 ns\n through 50 frames 16588.5 ns\n" }, { "name": "e05-24", @@ -711,7 +711,7 @@ { "name": "e05-29", "status": "ok", - "printed": " for loop table 0 bytes 73.4 ns per item\n while and next table 12 bytes 96.0 ns per item\n" + "printed": " for loop table 0 bytes 86.0 ns per item\n while and next table 12 bytes 103.5 ns per item\n" }, { "name": "e05-32", @@ -780,7 +780,7 @@ { "name": "e06-31", "status": "ok", - "printed": " every value an int 54.1 ns per addition\n every value a float 47.1 ns per addition\n alternating between the two 87.6 ns per addition\n" + "printed": " every value an int 41.3 ns per addition\n every value a float 41.4 ns per addition\n alternating between the two 69.9 ns per addition\n" }, { "name": "e06-34", @@ -834,7 +834,7 @@ { "name": "e07-25", "status": "ok", - "printed": " with the jit on\n this build has no JIT in it, so there is nothing to switch on\n\n with the jit off, for comparison\n arithmetic loop 174.0 ns per iteration\n loop with a call 203.4 ns per iteration\n" + "printed": " with the jit on\n this build has no JIT in it, so there is nothing to switch on\n\n with the jit off, for comparison\n arithmetic loop 116.0 ns per iteration\n loop with a call 147.7 ns per iteration\n" } ] }, @@ -863,7 +863,7 @@ { "name": "e08-11", "status": "ok", - "printed": " nobody watching 109.2 ns per turn\n callback counts the line 1042.7 ns per turn, 2000015 calls\n callback returns DISABLE 195.6 ns per turn\n" + "printed": " nobody watching 82.7 ns per turn\n callback counts the line 527.1 ns per turn, 2000015 calls\n callback returns DISABLE 111.4 ns per turn\n" }, { "name": "e08-14", @@ -1020,17 +1020,17 @@ { "name": "e11-15", "status": "ok", - "printed": " extra lines instructions run nanoseconds cost of one more\n 0 7010 57260 \n 5 17010 91379 3.41\n 10 27010 340889 24.95\n 20 47010 529148 9.41\n 40 87010 862085 8.32\n" + "printed": " extra lines instructions run nanoseconds cost of one more\n 0 7010 50629 \n 5 17010 80532 2.99\n 10 27010 268083 18.76\n 20 47010 433088 8.25\n 40 87010 776699 8.59\n" }, { "name": "e11-18", "status": "ok", - "printed": " the same job instructions nanoseconds each\n a while loop 20012 367055 18.34\n a for loop 10008 139560 13.94\n a comprehension 6011 112854 18.77\n" + "printed": " the same job instructions nanoseconds each\n a while loop 20012 155552 7.77\n a for loop 10008 70577 7.05\n a comprehension 6011 55464 9.23\n" }, { "name": "e11-21", "status": "ok", - "printed": " one bytecode instruction 9.41 ns\n one call to an empty def 46.61 ns beyond the code that calls it\n so a call costs about 5 instructions\n" + "printed": " one bytecode instruction 7.50 ns\n one call to an empty def 44.30 ns beyond the code that calls it\n so a call costs about 6 instructions\n" } ] }, @@ -1489,7 +1489,7 @@ { "name": "f10-11", "status": "ok", - "printed": " -- MAKE_CELL 0 (a)\n MAKE_CELL 6 (n)\n\n 1 RESUME 0\n\n 2 LOAD_SMALL_INT 0\n STORE_DEREF 6 (n)\n\n 4 LOAD_FAST_BORROW 0 (a)\n LOAD_FAST_BORROW 6 (n)\n BUILD_TUPLE 2\n LOAD_CONST 1 ()\n MAKE_FUNCTION\n SET_FUNCTION_ATTRIBUTE 8 (closure)\n STORE_FAST 5 (inner)\n\n 7 LOAD_FAST_BORROW 5 (inner)\n RETURN_VALUE\n\nDisassembly of :\n -- COPY_FREE_VARS 2\n\n 4 RESUME 0\n\n 5 LOAD_DEREF 0 (a)\n LOAD_DEREF 1 (n)\n BINARY_OP 0 (+)\n RETURN_VALUE\n" + "printed": " -- MAKE_CELL 0 (a)\n MAKE_CELL 6 (n)\n\n 1 RESUME 0\n\n 2 LOAD_SMALL_INT 0\n STORE_DEREF 6 (n)\n\n 4 LOAD_FAST_BORROW 0 (a)\n LOAD_FAST_BORROW 6 (n)\n BUILD_TUPLE 2\n LOAD_CONST 1 ()\n MAKE_FUNCTION\n SET_FUNCTION_ATTRIBUTE 8 (closure)\n STORE_FAST 5 (inner)\n\n 7 LOAD_FAST_BORROW 5 (inner)\n RETURN_VALUE\n\nDisassembly of :\n -- COPY_FREE_VARS 2\n\n 4 RESUME 0\n\n 5 LOAD_DEREF 0 (a)\n LOAD_DEREF 1 (n)\n BINARY_OP 0 (+)\n RETURN_VALUE\n" }, { "name": "f10-13", @@ -1581,12 +1581,12 @@ { "name": "f12-07", "status": "ok", - "printed": " greet.py is 54 bytes of text\n greet.cpython-314.pyc is 251 bytes\n\n header 2b 0e 0d 0a 00 00 00 00 63 a0 9c 6a 36 00 00 00\n the rest 235 bytes, and marshal.loads gives back a\n code object\n" + "printed": " greet.py is 54 bytes of text\n greet.cpython-314.pyc is 251 bytes\n\n header 2b 0e 0d 0a 00 00 00 00 6c a7 9c 6a 36 00 00 00\n the rest 235 bytes, and marshal.loads gives back a\n code object\n" }, { "name": "f12-10", "status": "ok", - "printed": " magic 168627755\n flags 0\n mtime 1788649571 source 1788649571\n size 54 source 54\n" + "printed": " magic 168627755\n flags 0\n mtime 1788651372 source 1788651372\n size 54 source 54\n" }, { "name": "f12-13", @@ -1620,7 +1620,7 @@ { "name": "f12-29", "status": "ok", - "printed": " one bit flipped in the magic: bad magic number in 'broken': b'*\\x0e\\r\\n'\n\n put a wrong mtime in greet.cpython-314.pyc: 1788649628\n imported it with no complaint: 'hi world'\n and the header now says 1788649628\n the file was quietly rewritten: False\n" + "printed": " one bit flipped in the magic: bad magic number in 'broken': b'*\\x0e\\r\\n'\n\n put a wrong mtime in greet.cpython-314.pyc: 1788651411\n imported it with no complaint: 'hi world'\n and the header now says 1788651411\n the file was quietly rewritten: False\n" } ] }, @@ -2002,7 +2002,7 @@ { "name": "m08-17", "status": "ok", - "printed": " cycles on the heap 200000\n one full pass over it 66 ms\n and over an empty one 24 ms\n" + "printed": " cycles on the heap 200000\n one full pass over it 48 ms\n and over an empty one 14 ms\n" }, { "name": "m08-20", @@ -2085,7 +2085,7 @@ { "name": "o01-07", "status": "ok", - "printed": " a pointer here is 4 bytes, so the header is 8\n it lives at 23966208\n\n first word, read as a count 1\n sys.getrefcount, less its own argument 1\n the same number: True\n\n second word 19430888\n id(Thing) is 19430888\n so the second word is the type pointer: True\n\n after a second name, the count reads 2\n after deleting that name, it reads 1\n" + "printed": " a pointer here is 4 bytes, so the header is 8\n it lives at 20521584\n\n first word, read as a count 1\n sys.getrefcount, less its own argument 1\n the same number: True\n\n second word 23961880\n id(Thing) is 23961880\n so the second word is the type pointer: True\n\n after a second name, the count reads 2\n after deleting that name, it reads 1\n" }, { "name": "o01-10", @@ -2958,7 +2958,7 @@ { "name": "r03-22", "status": "ok", - "printed": " first time, source file, no .pyc yet: 393.0 microseconds\n from a file, with a .pyc already there: 198.7 microseconds\n asking again for one already loaded: 302.2 nanoseconds\n the second of those is 657 times the third\n" + "printed": " first time, source file, no .pyc yet: 289.5 microseconds\n from a file, with a .pyc already there: 361.9 microseconds\n asking again for one already loaded: 303.9 nanoseconds\n the second of those is 1191 times the third\n" } ] }, @@ -3007,7 +3007,7 @@ { "name": "r04-25", "status": "ok", - "printed": " asking FrozenImporter for os: 3.31 microseconds\n asking PathFinder for the same name: 13311.16 microseconds\n this runtime keeps no pyc for os, so the last two rows have nothing to measure\n" + "printed": " asking FrozenImporter for os: 3.17 microseconds\n asking PathFinder for the same name: 14421.43 microseconds\n this runtime keeps no pyc for os, so the last two rows have nothing to measure\n" }, { "name": "r04-29", @@ -3398,7 +3398,7 @@ { "name": "t04-30", "status": "ok", - "printed": "free variables: ('total',)\nthe cells: (,)\nfirst call: 1\nsecond call: 2\nthe cell now: 2\n" + "printed": "free variables: ('total',)\nthe cells: (,)\nfirst call: 1\nsecond call: 2\nthe cell now: 2\n" }, { "name": "t04-33", @@ -3675,7 +3675,7 @@ { "name": "t07-38", "status": "ok", - "printed": "asked twice, got the same object: True\nand it is still here after the call returned: \nits name: make_one\n" + "printed": "asked twice, got the same object: True\nand it is still here after the call returned: \nits name: make_one\n" }, { "name": "t07-40", @@ -3709,12 +3709,12 @@ { "name": "t08-11", "status": "ok", - "printed": "NoneType at 0x30d7d8, refcount is parked, this object is never freed, 8 bytes, not tracked\nint at 0x31a7cc, refcount is parked, this object is never freed, 16 bytes, not tracked\nstr at 0x1680ac8, 3 reference(s), 26 bytes, not tracked\nlist at 0x15806d0, 2 reference(s), 44 bytes, tracked by the cycle collector\ndict at 0x17116f8, 2 reference(s), 108 bytes, tracked by the cycle collector\nfunction at 0x129db48, 4 reference(s), 84 bytes, tracked by the cycle collector\n" + "printed": "NoneType at 0x30d7d8, refcount is parked, this object is never freed, 8 bytes, not tracked\nint at 0x31a7cc, refcount is parked, this object is never freed, 16 bytes, not tracked\nstr at 0x137f100, 3 reference(s), 26 bytes, not tracked\nlist at 0x178dad8, 2 reference(s), 44 bytes, tracked by the cycle collector\ndict at 0x1717f48, 2 reference(s), 108 bytes, tracked by the cycle collector\nfunction at 0x18646c8, 4 reference(s), 84 bytes, tracked by the cycle collector\n" }, { "name": "t08-14", "status": "ok", - "printed": "a == b True same contents\na is b False different objects\na is c True same object\n\nid(a) 0x16025c0\nid(b) 0x17dc300\nid(c) 0x16025c0\n" + "printed": "a == b True same contents\na is b False different objects\na is c True same object\n\nid(a) 0x11d8f50\nid(b) 0x1202498\nid(c) 0x11d8f50\n" }, { "name": "t08-17", @@ -3754,7 +3754,7 @@ { "name": "t08-35", "status": "ok", - "printed": "dict at 0x13eb218\ndict at 0x1767720\nlist at 0x181e0f0\n" + "printed": "dict at 0x13e8a18\ndict at 0x1697428\nlist at 0x1202410\n" }, { "name": "t08-37", @@ -3813,7 +3813,7 @@ { "name": "t09-21", "status": "ok", - "printed": "Node at 0x18355a0 -> Node at 0x14bdf08 -> Node at 0x17d45c0 -> Node at 0x18355a0\n" + "printed": "Node at 0x1706cb0 -> Node at 0x1397848 -> Node at 0x18bdcb8 -> Node at 0x1706cb0\n" }, { "name": "t09-23", @@ -3843,7 +3843,7 @@ { "name": "t09-37", "status": "ok", - "printed": "first object was at 0x132d928\nsecond object is at 0x132d928\nsame address reused -> True\n" + "printed": "first object was at 0x1744da0\nsecond object is at 0x1744da0\nsame address reused -> True\n" } ] },