Update page 'Day 9 homework'
+177
@@ -0,0 +1,177 @@
|
||||
Today's homework will be to build a tool to introspect Python packages, from the perspective of the currently running Python interpreter.
|
||||
|
||||
**1. Use `importlib.util.find_spec` to create a function that takes a Python module or package name, and returns the total number of bytes in the cached `.pyc` file, or the total number of bytes in the `.py` file if there is no cached bytecode. You can use the os.stat command to get the size of a file.**
|
||||
|
||||
```
|
||||
import importlib.util
|
||||
import os
|
||||
|
||||
def get_module_size(module_name):
|
||||
spec = importlib.util.find_spec(module_name)
|
||||
|
||||
if spec is not None and spec.has_location:
|
||||
# If the module is found and has a location, check for .pyc file first
|
||||
pyc_file_path = spec.origin + 'c'
|
||||
if os.path.exists(pyc_file_path):
|
||||
return os.stat(pyc_file_path).st_size
|
||||
else:
|
||||
# If .pyc file doesn't exist, check for .py file
|
||||
py_file_path = spec.origin
|
||||
if os.path.exists(py_file_path):
|
||||
return os.stat(py_file_path).st_size
|
||||
|
||||
return 0 # Return 0 if no .pyc or .py file found
|
||||
```
|
||||
|
||||
**2. Create another function that uses the last function to see how many bytes are in all the modules listed in sys.modules.**
|
||||
|
||||
```
|
||||
import sys
|
||||
|
||||
def get_total_module_size():
|
||||
total_size = 0
|
||||
for module_name in sys.modules:
|
||||
module_size = get_module_size(module_name)
|
||||
total_size += module_size
|
||||
|
||||
return total_size
|
||||
|
||||
if __name__ == '__main__' and __package__ is None:
|
||||
__spec__ = importlib.util.spec_from_file_location('__main__', __file__)
|
||||
total_size = get_total_module_size()
|
||||
print("Total size of all modules in sys.modules: {} bytes".format(total_size))
|
||||
```
|
||||
|
||||
**3. Run your Python interpreter like you would normally, using Jupyter, and run the last function. How many bytes are being used?**
|
||||
I run this script in visual studio and change the python interpreter path including the defualt python3.11, the one in my anaconda environment, and the pandas environment. It seems when I use a python interpreter that ignores environment variables or only loads the modules that came with the original build of the Python interpreter, the total size of all modules in `sys.modules` would likely be relatively small compared to a Python environment with additional third-party modules installed (just like my anaconda environment, where I have intalled third-party modules such as fastapi).
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
**4. Now, run the same function, but start the Python interpreter in Powershell (Windows), or the regular shell (MacOS/Linux) with the `-E` flag, which ignores environment variables and should only load the modules that came with the original build of the Python interpreter. How many bytes are being used now?**
|
||||
I run the same function, but a warning says name '__file__' is not defined. The reason for this may be the fact that the `__file__` attribute is not automatically defined in the global namespace when running a script in Powershell (and I am not sure whether this issue is caused by my system's setting). So, to fix it, I just update my last function like this:
|
||||
|
||||
```
|
||||
import inspect
|
||||
|
||||
if __name__ == '__main__' and __package__ is None:
|
||||
# Get the path of the currently running script
|
||||
script_path = inspect.getframeinfo(inspect.currentframe()).filename
|
||||
__spec__ = importlib.util.spec_from_loader('__main__', None, origin=script_path)
|
||||
|
||||
total_size = get_total_module_size()
|
||||
print("Total size of all modules in sys.modules: {} bytes".format(total_size))
|
||||
```
|
||||
The result says that the total size of all modules in sys.modules: 671553 bytes.
|
||||
|
||||
**5. With only the `sys` module loaded, look at the `sys.modules` dict. What is the type of the value of the `os` key?**
|
||||
|
||||
```
|
||||
<class 'module'>
|
||||
```
|
||||
|
||||
**6. Import the os module. What happens when you type `type(os)`?**
|
||||
It returns the type of the `os` module as `<class 'module'>`.
|
||||
|
||||
**7. Restart the Python interpreter. Can you figure out how to call `os.getlogin()` without saying `import os`?**
|
||||
|
||||
```
|
||||
import builtins
|
||||
|
||||
# Access the os module from globals()
|
||||
os = globals().get('os', None)
|
||||
|
||||
# Check if os module is available
|
||||
if os is not None:
|
||||
# Call os.getlogin()
|
||||
username = os.getlogin()
|
||||
print("Username:", username)
|
||||
else:
|
||||
print("os module not available")
|
||||
```
|
||||
|
||||
**8. If you can, explain why this works and quote the relevant sections of section 5 in the language reference.**
|
||||
"Functions such as importlib.import_module() and built-in __import__() can also be used to invoke the import machinery."
|
||||
"When a module is first imported, Python searches for the module and if found, it creates a module object 1, initializing it.
|
||||
"when the name binding operation is executed, the Python interpreter searches several namespaces, in this order, to find the specified name: the local namespace, the global namespace, the built-in namespace."
|
||||
|
||||
**9. Using the functions you created above and `sys.modules`, create a module designed to be run from the command line (`python -m myscript.py`) that generates a report summarizing the module environment, and break down the percentages of bytecode modules, DSO modules (these have `file` end in `.so` or `.dll`), and normal python code.**
|
||||
|
||||
```
|
||||
import os
|
||||
import sys
|
||||
import importlib.util
|
||||
import types
|
||||
|
||||
|
||||
def get_bytecode_size(module_name):
|
||||
"""Get the size of the cached .pyc bytecode file for a module."""
|
||||
spec = importlib.util.find_spec(module_name)
|
||||
if spec and spec.origin.endswith('.py'):
|
||||
pyc_file = importlib.util.cache_from_source(spec.origin)
|
||||
if os.path.exists(pyc_file):
|
||||
return os.path.getsize(pyc_file)
|
||||
return 0
|
||||
|
||||
|
||||
def get_dso_size(module_name):
|
||||
"""Get the size of the DSO (Dynamic Shared Object) module file for a module."""
|
||||
spec = importlib.util.find_spec(module_name)
|
||||
if spec and spec.origin.endswith(('.so', '.dll')):
|
||||
return os.path.getsize(spec.origin)
|
||||
return 0
|
||||
|
||||
|
||||
def generate_module_report():
|
||||
"""Generate a report summarizing the module environment."""
|
||||
total_modules = len(sys.modules)
|
||||
bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
|
||||
dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
|
||||
python_size = sum(
|
||||
sys.modules[module_name].__sizeof__()
|
||||
for module_name in sys.modules
|
||||
if isinstance(sys.modules[module_name], types.ModuleType)
|
||||
)
|
||||
total_size = bytecode_size + dso_size + python_size
|
||||
|
||||
# Calculate percentage of each type of module
|
||||
bytecode_percent = bytecode_size / total_size * 100
|
||||
dso_percent = dso_size / total_size * 100
|
||||
python_percent = python_size / total_size * 100
|
||||
|
||||
# Generate the report
|
||||
report = f"Total modules: {total_modules}\n"
|
||||
report += f"Total size: {total_size} bytes\n"
|
||||
report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
|
||||
report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
|
||||
report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
|
||||
|
||||
return report
|
||||
|
||||
if __name__ == '__main__' and __package__ is None:
|
||||
__spec__ = importlib.util.spec_from_file_location('__main__', __file__)
|
||||
report = generate_module_report()
|
||||
print(report)
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
**10. Add a feature to the script which looks at `file` for each module, trims the filename to get the directory name, then prints out a histogram of how many modules are in each directory name. If you can't find a text histogram output library in anaconda's built-in modules, just print out the `*` character for each module.**
|
||||
|
||||
```
|
||||
# Generate histogram of module directory names
|
||||
dir_histogram = defaultdict(int)
|
||||
for module_name in sys.modules:
|
||||
module_file = sys.modules[module_name].__file__
|
||||
if module_file:
|
||||
module_dir = os.path.dirname(module_file)
|
||||
dir_histogram[module_dir] += 1
|
||||
|
||||
report += "\nModule Directory Histogram:\n"
|
||||
for module_dir, count in dir_histogram.items():
|
||||
report += f"{module_dir}: {'*' * count}\n"
|
||||
```
|
||||
Reference in New Issue
Block a user