diff options
author | Dave Cobbley <david.j.cobbley@linux.intel.com> | 2018-08-14 20:05:37 +0300 |
---|---|---|
committer | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2018-08-23 04:26:31 +0300 |
commit | eb8dc40360f0cfef56fb6947cc817a547d6d9bc6 (patch) | |
tree | de291a73dc37168da6370e2cf16c347d1eba9df8 /poky/meta/recipes-devtools/python/python3/create_manifest3.py | |
parent | 9c3cf826d853102535ead04cebc2d6023eff3032 (diff) | |
download | openbmc-eb8dc40360f0cfef56fb6947cc817a547d6d9bc6.tar.xz |
[Subtree] Removing import-layers directory
As part of the move to subtrees, need to bring all the import layers
content to the top level.
Change-Id: I4a163d10898cbc6e11c27f776f60e1a470049d8f
Signed-off-by: Dave Cobbley <david.j.cobbley@linux.intel.com>
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Diffstat (limited to 'poky/meta/recipes-devtools/python/python3/create_manifest3.py')
-rw-r--r-- | poky/meta/recipes-devtools/python/python3/create_manifest3.py | 354 |
1 files changed, 354 insertions, 0 deletions
diff --git a/poky/meta/recipes-devtools/python/python3/create_manifest3.py b/poky/meta/recipes-devtools/python/python3/create_manifest3.py new file mode 100644 index 000000000..2f944f9b1 --- /dev/null +++ b/poky/meta/recipes-devtools/python/python3/create_manifest3.py @@ -0,0 +1,354 @@ +# This script is used as a bitbake task to create a new python manifest +# $ bitbake python -c create_manifest +# +# Our goal is to keep python-core as small as posible and add other python +# packages only when the user needs them, hence why we split upstream python +# into several packages. +# +# In a very simplistic way what this does is: +# Launch python and see specifically what is required for it to run at a minimum +# +# Go through the python-manifest file and launch a separate task for every single +# one of the files on each package, this task will check what was required for that +# specific module to run, these modules will be called dependencies. +# The output of such task will be a list of the modules or dependencies that were +# found for that file. +# +# Such output will be parsed by this script, we will look for each dependency on the +# manifest and if we find that another package already includes it, then we will add +# that package as an RDEPENDS to the package we are currently checking; in case we dont +# find the current dependency on any other package we will add it to the current package +# as part of FILES. +# +# +# This way we will create a new manifest from the data structure that was built during +# this process, ont this new manifest each package will contain specifically only +# what it needs to run. +# +# There are some caveats which we try to deal with, such as repeated files on different +# packages, packages that include folders, wildcards, and special packages. +# Its also important to note that this method only works for python files, and shared +# libraries. Static libraries, header files and binaries need to be dealt with manually. +# +# This script differs from its python2 version mostly on how shared libraries are handled +# The manifest file for python3 has an extra field which contains the cached files for +# each package. +# Tha method to handle cached files does not work when a module includes a folder which +# itself contains the pycache folder, gladly this is almost never the case. +# +# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com> + + +import sys +import subprocess +import json +import os + +# Hack to get native python search path (for folders), not fond of it but it works for now +pivot='recipe-sysroot-native' +for p in sys.path: + if pivot in p: + nativelibfolder=p[:p.find(pivot)+len(pivot)] + +# Empty dict to hold the whole manifest +new_manifest = {} + +# Check for repeated files, folders and wildcards +allfiles=[] +repeated=[] +wildcards=[] + +hasfolders=[] +allfolders=[] + +def isFolder(value): + if os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib64')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib32')): + return True + else: + return False + +def isCached(item): + if '__pycache__' in item: + return True + else: + return False + +# Read existing JSON manifest +with open('python3-manifest.json') as manifest: + old_manifest=json.load(manifest) + + +# First pass to get core-package functionality, because we base everything on the fact that core is actually working +# Not exactly the same so it should not be a function +print ('Getting dependencies for package: core') + +# Special call to check for core package +output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package']).decode('utf8') +for item in output.split(): + # We append it so it doesnt hurt what we currently have: + if isCached(item): + if item not in old_manifest['core']['cached']: + # We use the same data structure since its the one which will be used to check + # dependencies for other packages + old_manifest['core']['cached'].append(item) + else: + if item not in old_manifest['core']['files']: + # We use the same data structure since its the one which will be used to check + # dependencies for other packages + old_manifest['core']['files'].append(item) + +for value in old_manifest['core']['files']: + # Ignore folders, since we don't import those, difficult to handle multilib + if isFolder(value): + # Pass it directly + if isCached(value): + if value not in old_manifest['core']['cached']: + old_manifest['core']['cached'].append(value) + else: + if value not in old_manifest['core']['files']: + old_manifest['core']['files'].append(value) + continue + # Ignore binaries, since we don't import those, assume it was added correctly (manually) + if '${bindir}' in value: + # Pass it directly + if value not in old_manifest['core']['files']: + old_manifest['core']['files'].append(value) + continue + # Ignore empty values + if value == '': + continue + if '${includedir}' in value: + if value not in old_manifest['core']['files']: + old_manifest['core']['files'].append(value) + continue + # Get module name , shouldnt be affected by libdir/bindir + value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0] + + # Launch separate task for each module for deterministic behavior + # Each module will only import what is necessary for it to work in specific + print ('Getting dependencies for module: %s' % value) + output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % value]).decode('utf8') + print ('The following dependencies were found for module %s:\n' % value) + print (output) + for item in output.split(): + # We append it so it doesnt hurt what we currently have: + if isCached(item): + if item not in old_manifest['core']['cached']: + # We use the same data structure since its the one which will be used to check + # dependencies for other packages + old_manifest['core']['cached'].append(item) + else: + if item not in old_manifest['core']['files']: + # We use the same data structure since its the one which will be used to check + # dependencies for other packages + old_manifest['core']['files'].append(item) + + +# We check which packages include folders +for key in old_manifest: + for value in old_manifest[key]['files']: + # Ignore folders, since we don't import those, difficult to handle multilib + if isFolder(value): + print ('%s is a folder' % value) + if key not in hasfolders: + hasfolders.append(key) + if value not in allfolders: + allfolders.append(value) + +for key in old_manifest: + # Use an empty dict as data structure to hold data for each package and fill it up + new_manifest[key]={} + new_manifest[key]['files']=[] + + new_manifest[key]['rdepends']=[] + # All packages should depend on core + if key != 'core': + new_manifest[key]['rdepends'].append('core') + new_manifest[key]['cached']=[] + else: + new_manifest[key]['cached']=old_manifest[key]['cached'] + new_manifest[key]['summary']=old_manifest[key]['summary'] + + # Handle special cases, we assume that when they were manually added + # to the manifest we knew what we were doing. + print('\n') + print('--------------------------') + print ('Handling package %s' % key) + print('--------------------------') + special_packages=['misc', 'modules', 'dev'] + if key in special_packages or 'staticdev' in key: + print('Passing %s package directly' % key) + new_manifest[key]=old_manifest[key] + continue + + for value in old_manifest[key]['files']: + # We already handled core on the first pass + if key == 'core': + new_manifest[key]['files'].append(value) + continue + # Ignore folders, since we don't import those, difficult to handle multilib + if isFolder(value): + # Pass folders directly + new_manifest[key]['files'].append(value) + # Ignore binaries, since we don't import those + if '${bindir}' in value: + # Pass it directly to the new manifest data structure + if value not in new_manifest[key]['files']: + new_manifest[key]['files'].append(value) + continue + # Ignore empty values + if value == '': + continue + if '${includedir}' in value: + if value not in new_manifest[key]['files']: + new_manifest[key]['files'].append(value) + continue + + # Get module name , shouldnt be affected by libdir/bindir + # We need to check if the imported module comes from another (e.g. sqlite3.dump) + path,value = os.path.split(value) + path = os.path.basename(path) + value = os.path.splitext(os.path.basename(value))[0] + + # If this condition is met, it means we need to import it from another module + # or its the folder itself (e.g. unittest) + if path == key: + if value: + value = path + '.' + value + else: + value = path + + # Launch separate task for each module for deterministic behavior + # Each module will only import what is necessary for it to work in specific + print ('\nGetting dependencies for module: %s' % value) + output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % value]).decode('utf8') + # We can print dependencies for debugging purposes + print ('The following dependencies were found for module %s:\n' % value) + print (output) + # Output will have all dependencies + + reportFILES = [] + reportRDEPS = [] + + for item in output.split(): + + # Warning: This first part is ugly + # One of the dependencies that was found, could be inside of one of the folders included by another package + # We need to check if this happens so we can add the package containing the folder as an rdependency + # e.g. Folder encodings contained in codecs + # This would be solved if no packages included any folders + + # This can be done in two ways: + # 1 - We assume that if we take out the filename from the path we would get + # the folder string, then we would check if folder string is in the list of folders + # This would not work if a package contains a folder which contains another folder + # e.g. path/folder1/folder2/filename folder_string= path/folder1/folder2 + # folder_string would not match any value contained in the list of folders + # + # 2 - We do it the other way around, checking if the folder is contained in the path + # e.g. path/folder1/folder2/filename folder_string= path/folder1/folder2 + # is folder_string inside path/folder1/folder2/filename?, + # Yes, it works, but we waste a couple of milliseconds. + + inFolders=False + for folder in allfolders: + if folder in item: + inFolders = True # Did we find a folder? + folderFound = False # Second flag to break inner for + # Loop only through packages which contain folders + for keyfolder in hasfolders: + if (folderFound == False): + #print('Checking folder %s on package %s' % (item,keyfolder)) + for file_folder in old_manifest[keyfolder]['files'] or file_folder in old_manifest[keyfolder]['cached']: + if file_folder==folder: + print ('%s folder found in %s' % (folder, keyfolder)) + folderFound = True + if keyfolder not in new_manifest[key]['rdepends'] and keyfolder != key: + new_manifest[key]['rdepends'].append(keyfolder) + + else: + break + + # A folder was found so we're done with this item, we can go on + if inFolders: + continue + + + # We might already have it on the dictionary since it could depend on a (previously checked) module + if item not in new_manifest[key]['files'] and item not in new_manifest[key]['cached']: + # Handle core as a special package, we already did it so we pass it to NEW data structure directly + if key=='core': + print('Adding %s to %s FILES' % (item, key)) + if item.endswith('*'): + wildcards.append(item) + if isCached(item): + new_manifest[key]['cached'].append(item) + else: + new_manifest[key]['files'].append(item) + + # Check for repeated files + if item not in allfiles: + allfiles.append(item) + else: + repeated.append(item) + + else: + + + # Check if this dependency is already contained on another package, so we add it + # as an RDEPENDS, or if its not, it means it should be contained on the current + # package, so we should add it to FILES + for newkey in old_manifest: + # Debug + #print('Checking %s ' % item + ' in %s' % newkey) + if item in old_manifest[newkey]['files'] or item in old_manifest[newkey]['cached']: + # Since were nesting, we need to check its not the same key + if(newkey!=key): + if newkey not in new_manifest[key]['rdepends']: + # Add it to the new manifest data struct + reportRDEPS.append('Adding %s to %s RDEPENDS, because it contains %s\n' % (newkey, key, item)) + new_manifest[key]['rdepends'].append(newkey) + break + else: + # A module shouldn't contain itself (${libdir}/python3/sqlite3 shouldnt be on sqlite3 files) + if os.path.basename(item) != key: + reportFILES.append(('Adding %s to %s FILES\n' % (item, key))) + # Since it wasnt found on another package, its not an RDEP, so add it to FILES for this package + if isCached(item): + new_manifest[key]['cached'].append(item) + else: + new_manifest[key]['files'].append(item) + + if item.endswith('*'): + wildcards.append(item) + if item not in allfiles: + allfiles.append(item) + else: + repeated.append(item) + + print('\n') + print('#################################') + print('Summary for module %s' % value) + print('FILES found for module %s:' % value) + print(''.join(reportFILES)) + print('RDEPENDS found for module %s:' % value) + print(''.join(reportRDEPS)) + print('#################################') + +print ('The following files are repeated (contained in more than one package), please check which package should get it:') +print (repeated) +print('The following files contain wildcards, please check they are necessary') +print(wildcards) +print('The following files contain folders, please check they are necessary') +print(hasfolders) + +# Sort it just so it looks nicer +for key in new_manifest: + new_manifest[key]['files'].sort() + new_manifest[key]['cached'].sort() + new_manifest[key]['rdepends'].sort() + +# Create the manifest from the data structure that was built +with open('python3-manifest.json.new','w') as outfile: + json.dump(new_manifest,outfile,sort_keys=True, indent=4) + outfile.write('\n') |