@@ -58,6 +58,7 @@ essential_libraries = {
5858 'accelerate' : True
5959}
6060
61+ # Collect all libraries using collect_all (includes data files and binaries)
6162for lib in essential_libraries :
6263 try :
6364 lib_datas , lib_binaries , lib_hiddenimports = collect_all (lib )
@@ -67,6 +68,25 @@ for lib in essential_libraries:
6768 except Exception :
6869 pass
6970
71+ # Additionally collect ALL submodules for libraries that commonly have dynamic imports
72+ # This is a more aggressive approach but ensures we don't miss any modules
73+ # Libraries that are known to have many dynamic imports and submodules
74+ libraries_with_dynamic_imports = [
75+ 'scipy' , # Has many subpackages: stats, interpolate, optimize, linalg, sparse, signal, etc.
76+ 'sklearn' , # Has many submodules that may be dynamically imported
77+ 'transformers' , # Has dynamic model loading
78+ 'torch' , # Has many submodules, especially _dynamo.polyfills
79+ ]
80+
81+ # Collect all submodules for these libraries to ensure comprehensive coverage
82+ for lib in libraries_with_dynamic_imports :
83+ try :
84+ submodules = collect_submodules (lib )
85+ all_hiddenimports .extend (submodules )
86+ print (f"Collected { len (submodules )} submodules from { lib } " )
87+ except Exception as e :
88+ print (f"Warning: Failed to collect submodules from { lib } : { e } " )
89+
7090
7191# Helper function to collect submodules with fallback
7292def collect_submodules_with_fallback (package , fallback_modules = None , package_name = None ):
@@ -91,10 +111,13 @@ def collect_submodules_with_fallback(package, fallback_modules=None, package_nam
91111 print (f"Using fallback modules for { package_name } " )
92112
93113
94- # Packages that need submodule collection due to dynamic imports
114+ # Additional specific packages that need submodule collection
115+ # Note: scipy, sklearn, transformers, torch are already collected above via libraries_with_dynamic_imports
116+ # This section is for more specific sub-packages that need special handling
95117# Format: (package_name, fallback_modules_list, display_name)
96118submodule_collection_configs = [
97119 # torch._dynamo.polyfills - critical for torch dynamo functionality
120+ # (torch is already collected above, but this ensures polyfills are included)
98121 (
99122 'torch._dynamo.polyfills' ,
100123 [
@@ -105,22 +128,10 @@ submodule_collection_configs = [
105128 ],
106129 'torch._dynamo.polyfills'
107130 ),
108- # transformers packages with dynamic imports
131+ # transformers sub-packages with dynamic imports
132+ # (transformers is already collected above, but these specific sub-packages may need extra attention)
109133 ('transformers.generation' , None , 'transformers.generation' ),
110134 ('transformers.models.auto' , None , 'transformers.models.auto' ),
111- # scipy.stats - contains many private modules (starting with _)
112- (
113- 'scipy.stats' ,
114- [
115- 'scipy.stats._variation' ,
116- 'scipy.stats._morestats' ,
117- 'scipy.stats._stats' ,
118- 'scipy.stats._distn_infrastructure' ,
119- ],
120- 'scipy.stats'
121- ),
122- # sklearn - has many submodules that may be dynamically imported
123- ('sklearn' , None , 'sklearn' ),
124135]
125136
126137# Collect submodules for all configured packages
0 commit comments