From 44e7920d25b6024efcc7c4d84c406a2030d86ee3 Mon Sep 17 00:00:00 2001 From: Karenzz Date: Wed, 19 Apr 2023 23:51:35 +0200 Subject: [PATCH] Update page 'Day10 homework' --- Day10 homework.md | 132 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 Day10 homework.md diff --git a/Day10 homework.md b/Day10 homework.md new file mode 100644 index 0000000..a6bfcef --- /dev/null +++ b/Day10 homework.md @@ -0,0 +1,132 @@ +This is an extension to the day 9 homework. +First, rebase your branch onto the main branch of this repository: `git rebase main`. + +**1. Run the module-inspector script, and fix any problems that prevent it from running.** +**2. Look at the output. Notice that some of the module names seem similar, and are cached via two different loader methods.** +**3. Based on what you see from the module-inspector script, modify your answer from question 10 from the day 9 homework, so the similar module names aren't counted multiple times.** + +``` +import os +import sys +import importlib.util +import types +from collections import defaultdict +from sys import getsizeof + + +def get_bytecode_size(module_name): + """Get the size of the cached .pyc bytecode file for a module.""" + spec = importlib.util.find_spec(module_name) + if spec and spec.origin and spec.origin.endswith('.py'): + pyc_file = importlib.util.cache_from_source(spec.origin) + if os.path.exists(pyc_file): + return os.path.getsize(pyc_file) + return 0 + + +def get_dso_size(module_name): + """Get the size of the DSO (Dynamic Shared Object) module file for a module.""" + spec = importlib.util.find_spec(module_name) + if spec and spec.origin and spec.origin.endswith(('.so', '.dll')): + return os.path.getsize(spec.origin) + return 0 + + +def generate_module_report(): + """Generate a report summarizing the module environment.""" + total_modules = len(sys.modules) + bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules) + dso_size = sum(get_dso_size(module_name) for module_name in sys.modules) + python_size = sum( + getsizeof(sys.modules[module_name]) - getsizeof(module_name) + for module_name in sys.modules + if isinstance(sys.modules[module_name], types.ModuleType) + ) + total_size = bytecode_size + dso_size + python_size + + # Calculate percentage of each type of module + bytecode_percent = bytecode_size / total_size * 100 + dso_percent = dso_size / total_size * 100 + python_percent = python_size / total_size * 100 + + # Generate the report + report = f"Total modules: {total_modules}\n" + report += f"Total size: {total_size} bytes\n" + report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n" + report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n" + report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n" + + # Generate histogram of module directory names + dir_histogram = defaultdict(int) + for module_name in sys.modules: + module_file = getattr(sys.modules[module_name], '__file__', None) + if module_file: + module_dir = os.path.dirname(module_file) + dir_histogram[module_dir] += 1 + + report += "\nModule Directory Histogram:\n" + for module_dir, count in dir_histogram.items(): + report += f"{module_dir}: {'*' * count}\n" + + return report + + +if __name__ == "__main__": + __spec__ = importlib.util.spec_from_file_location('__main__', __file__) +report = generate_module_report() +print(report) +``` + + +![Finally, god](https://gogs.gtsplab.com:5432/j/python-bootcamp/raw/Branch_Karen/hw10.png) + + + +**4. Inspect the `loader` attribute of each entry in the `sys.modules` dict. Can you find an attribute of `loader` which reliably tells you if the loader is source, builtin, or extension?** + +From the module-inspector script, we can get lines for inspecting the `loader` attribute and make slight changes to have a script like this: + +``` +import sys + +for module_name, module in sys.modules.items(): + # Check if the module has a loader attribute + if hasattr(module, '__loader__'): + loader = module.__loader__ + # Determine the type of loader + if type(loader).__name__ == 'SourceFileLoader': + print(f"Module '{module_name}' has a source loader.") + elif type(loader).__name__ == 'BuiltinImporter': + print(f"Module '{module_name}' has a built-in loader.") + elif type(loader).__name__ == 'ExtensionFileLoader': + print(f"Module '{module_name}' has an extension loader.") +``` +The results vary when I use different python interpreter (here I try the default py3.11 and the anaconda). + + +![loader](https://gogs.gtsplab.com:5432/j/python-bootcamp/raw/Branch_Karen/loader.png) + + +![anaconda_loader](https://gogs.gtsplab.com:5432/j/python-bootcamp/raw/Branch_Karen/anocoda_loader.png) + + + +**5. Add `import pandas` to the top of the module-inspector script and re-run it. Notice that many of the pandas modules are not loaded using the `SourceFileLoader`. Can you explain why these modules are different?** + +I guess the reason why these moudles are different is that they are part of a compiled package, specifically including C extension modules such as `pandas._libs`. In this case, it may use `ExtensionFileLoader` rather than `SourceFileLoader`. + + +**Extra credit: use the freeze utility to turn the module-inspector script into a real app.** + +``` +from cx_Freeze import setup, Executable + +executables = [Executable('module-inspector.py')] + +setup( + name='wow', + version='1.0', + description='Module-inspector Application', + executables=executables +) +``` \ No newline at end of file