From 60f9d69e016b11c468c98ea75ba0a60c44afbbc4 Mon Sep 17 00:00:00 2001 From: Patrick Williams Date: Wed, 17 Aug 2016 14:31:25 -0500 Subject: yocto-poky: Move to import-layers subdir We are going to import additional layers, so create a subdir to hold all of the layers that we import with git-subtree. Change-Id: I6f732153a22be8ca663035c518837e3cc5ec0799 Signed-off-by: Patrick Williams --- import-layers/yocto-poky/bitbake/AUTHORS | 10 + import-layers/yocto-poky/bitbake/COPYING | 339 ++ import-layers/yocto-poky/bitbake/ChangeLog | 317 ++ import-layers/yocto-poky/bitbake/HEADER | 19 + import-layers/yocto-poky/bitbake/LICENSE | 12 + import-layers/yocto-poky/bitbake/bin/bitbake | 53 + .../yocto-poky/bitbake/bin/bitbake-diffsigs | 138 + .../yocto-poky/bitbake/bin/bitbake-dumpsig | 65 + .../yocto-poky/bitbake/bin/bitbake-layers | 1072 +++++++ .../yocto-poky/bitbake/bin/bitbake-prserv | 55 + .../yocto-poky/bitbake/bin/bitbake-selftest | 55 + .../yocto-poky/bitbake/bin/bitbake-worker | 447 +++ import-layers/yocto-poky/bitbake/bin/bitdoc | 531 ++++ import-layers/yocto-poky/bitbake/bin/image-writer | 122 + import-layers/yocto-poky/bitbake/bin/toaster | 276 ++ .../yocto-poky/bitbake/bin/toaster-eventreplay | 174 ++ import-layers/yocto-poky/bitbake/contrib/README | 1 + import-layers/yocto-poky/bitbake/contrib/bbdev.sh | 31 + .../yocto-poky/bitbake/contrib/dump_cache.py | 68 + .../bitbake/contrib/vim/ftdetect/bitbake.vim | 24 + .../bitbake/contrib/vim/ftplugin/bitbake.vim | 2 + .../bitbake/contrib/vim/plugin/newbb.vim | 84 + .../bitbake/contrib/vim/syntax/bitbake.vim | 126 + import-layers/yocto-poky/bitbake/doc/COPYING.GPL | 339 ++ import-layers/yocto-poky/bitbake/doc/COPYING.MIT | 17 + import-layers/yocto-poky/bitbake/doc/Makefile | 91 + import-layers/yocto-poky/bitbake/doc/README | 39 + .../bitbake-user-manual-customization.xsl | 29 + .../bitbake-user-manual-execution.xml | 931 ++++++ .../bitbake-user-manual-fetching.xml | 765 +++++ .../bitbake-user-manual-hello.xml | 505 +++ .../bitbake-user-manual-intro.xml | 691 +++++ .../bitbake-user-manual-metadata.xml | 1918 ++++++++++++ .../bitbake-user-manual-ref-variables.xml | 2319 ++++++++++++++ .../bitbake-user-manual-style.css | 984 ++++++ .../bitbake-user-manual/bitbake-user-manual.xml | 88 + .../bitbake-user-manual/figures/bitbake-title.png | Bin 0 -> 5086 bytes .../bitbake/doc/bitbake-user-manual/html.css | 281 ++ import-layers/yocto-poky/bitbake/doc/bitbake.1 | 142 + import-layers/yocto-poky/bitbake/doc/poky.ent | 59 + .../yocto-poky/bitbake/doc/template/Vera.ttf | Bin 0 -> 65932 bytes .../yocto-poky/bitbake/doc/template/Vera.xml | 1 + .../yocto-poky/bitbake/doc/template/VeraMoBd.ttf | Bin 0 -> 49052 bytes .../yocto-poky/bitbake/doc/template/VeraMoBd.xml | 1 + .../yocto-poky/bitbake/doc/template/VeraMono.ttf | Bin 0 -> 49224 bytes .../yocto-poky/bitbake/doc/template/VeraMono.xml | 1 + .../bitbake/doc/template/component.title.xsl | 39 + .../yocto-poky/bitbake/doc/template/db-pdf.xsl | 64 + .../bitbake/doc/template/division.title.xsl | 25 + .../yocto-poky/bitbake/doc/template/draft.png | Bin 0 -> 24847 bytes .../yocto-poky/bitbake/doc/template/fop-config.xml | 58 + .../bitbake/doc/template/formal.object.heading.xsl | 21 + .../bitbake/doc/template/gloss-permalinks.xsl | 14 + .../yocto-poky/bitbake/doc/template/permalinks.xsl | 25 + .../bitbake/doc/template/section.title.xsl | 55 + .../bitbake/doc/template/titlepage.templates.xml | 1259 ++++++++ .../yocto-poky/bitbake/doc/tools/docbook-to-pdf | 51 + import-layers/yocto-poky/bitbake/lib/bb/COW.py | 323 ++ .../yocto-poky/bitbake/lib/bb/__init__.py | 144 + import-layers/yocto-poky/bitbake/lib/bb/build.py | 784 +++++ import-layers/yocto-poky/bitbake/lib/bb/cache.py | 849 +++++ .../yocto-poky/bitbake/lib/bb/cache_extra.py | 75 + .../yocto-poky/bitbake/lib/bb/checksum.py | 139 + .../yocto-poky/bitbake/lib/bb/codeparser.py | 436 +++ import-layers/yocto-poky/bitbake/lib/bb/command.py | 474 +++ import-layers/yocto-poky/bitbake/lib/bb/compat.py | 6 + import-layers/yocto-poky/bitbake/lib/bb/cooker.py | 2196 +++++++++++++ .../yocto-poky/bitbake/lib/bb/cookerdata.py | 345 +++ .../yocto-poky/bitbake/lib/bb/daemonize.py | 193 ++ import-layers/yocto-poky/bitbake/lib/bb/data.py | 448 +++ .../yocto-poky/bitbake/lib/bb/data_smart.py | 969 ++++++ import-layers/yocto-poky/bitbake/lib/bb/event.py | 679 ++++ .../yocto-poky/bitbake/lib/bb/exceptions.py | 91 + .../yocto-poky/bitbake/lib/bb/fetch2/__init__.py | 1773 +++++++++++ .../yocto-poky/bitbake/lib/bb/fetch2/bzr.py | 143 + .../yocto-poky/bitbake/lib/bb/fetch2/clearcase.py | 263 ++ .../yocto-poky/bitbake/lib/bb/fetch2/cvs.py | 171 + .../yocto-poky/bitbake/lib/bb/fetch2/git.py | 435 +++ .../yocto-poky/bitbake/lib/bb/fetch2/gitannex.py | 76 + .../yocto-poky/bitbake/lib/bb/fetch2/gitsm.py | 134 + .../yocto-poky/bitbake/lib/bb/fetch2/hg.py | 278 ++ .../yocto-poky/bitbake/lib/bb/fetch2/local.py | 129 + .../yocto-poky/bitbake/lib/bb/fetch2/npm.py | 284 ++ .../yocto-poky/bitbake/lib/bb/fetch2/osc.py | 135 + .../yocto-poky/bitbake/lib/bb/fetch2/perforce.py | 187 ++ .../yocto-poky/bitbake/lib/bb/fetch2/repo.py | 98 + .../yocto-poky/bitbake/lib/bb/fetch2/sftp.py | 129 + .../yocto-poky/bitbake/lib/bb/fetch2/ssh.py | 128 + .../yocto-poky/bitbake/lib/bb/fetch2/svn.py | 197 ++ .../yocto-poky/bitbake/lib/bb/fetch2/wget.py | 565 ++++ import-layers/yocto-poky/bitbake/lib/bb/main.py | 440 +++ .../yocto-poky/bitbake/lib/bb/methodpool.py | 40 + .../yocto-poky/bitbake/lib/bb/monitordisk.py | 263 ++ import-layers/yocto-poky/bitbake/lib/bb/msg.py | 199 ++ .../bitbake/lib/bb/namedtuple_with_abc.py | 255 ++ .../yocto-poky/bitbake/lib/bb/parse/__init__.py | 170 + .../yocto-poky/bitbake/lib/bb/parse/ast.py | 476 +++ .../bitbake/lib/bb/parse/parse_py/BBHandler.py | 254 ++ .../bitbake/lib/bb/parse/parse_py/ConfHandler.py | 193 ++ .../bitbake/lib/bb/parse/parse_py/__init__.py | 33 + .../yocto-poky/bitbake/lib/bb/persist_data.py | 218 ++ import-layers/yocto-poky/bitbake/lib/bb/process.py | 156 + .../yocto-poky/bitbake/lib/bb/providers.py | 428 +++ .../yocto-poky/bitbake/lib/bb/pysh/__init__.py | 0 .../yocto-poky/bitbake/lib/bb/pysh/builtin.py | 710 +++++ .../yocto-poky/bitbake/lib/bb/pysh/interp.py | 1367 ++++++++ .../yocto-poky/bitbake/lib/bb/pysh/lsprof.py | 116 + .../yocto-poky/bitbake/lib/bb/pysh/pysh.py | 167 + .../yocto-poky/bitbake/lib/bb/pysh/pyshlex.py | 888 ++++++ .../yocto-poky/bitbake/lib/bb/pysh/pyshyacc.py | 779 +++++ .../yocto-poky/bitbake/lib/bb/pysh/sherrors.py | 41 + .../bitbake/lib/bb/pysh/subprocess_fix.py | 77 + .../yocto-poky/bitbake/lib/bb/runqueue.py | 2285 ++++++++++++++ .../yocto-poky/bitbake/lib/bb/server/__init__.py | 99 + .../yocto-poky/bitbake/lib/bb/server/process.py | 268 ++ .../yocto-poky/bitbake/lib/bb/server/xmlrpc.py | 390 +++ import-layers/yocto-poky/bitbake/lib/bb/shell.py | 820 +++++ import-layers/yocto-poky/bitbake/lib/bb/siggen.py | 601 ++++ .../yocto-poky/bitbake/lib/bb/taskdata.py | 690 +++++ .../yocto-poky/bitbake/lib/bb/tests/__init__.py | 0 .../yocto-poky/bitbake/lib/bb/tests/codeparser.py | 380 +++ .../yocto-poky/bitbake/lib/bb/tests/cow.py | 136 + .../yocto-poky/bitbake/lib/bb/tests/data.py | 446 +++ .../yocto-poky/bitbake/lib/bb/tests/fetch.py | 812 +++++ .../yocto-poky/bitbake/lib/bb/tests/parse.py | 147 + .../yocto-poky/bitbake/lib/bb/tests/utils.py | 603 ++++ import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py | 105 + .../yocto-poky/bitbake/lib/bb/ui/__init__.py | 17 + .../bitbake/lib/bb/ui/buildinfohelper.py | 1521 +++++++++ .../bitbake/lib/bb/ui/crumbs/__init__.py | 17 + .../bitbake/lib/bb/ui/crumbs/hig/__init__.py | 0 .../bitbake/lib/bb/ui/crumbs/hig/crumbsdialog.py | 44 + .../lib/bb/ui/crumbs/hig/crumbsmessagedialog.py | 70 + .../lib/bb/ui/crumbs/hig/deployimagedialog.py | 219 ++ .../lib/bb/ui/crumbs/hig/imageselectiondialog.py | 172 + .../lib/bb/ui/crumbs/hig/layerselectiondialog.py | 298 ++ .../bitbake/lib/bb/ui/crumbs/hig/propertydialog.py | 437 +++ .../lib/bb/ui/crumbs/hig/settingsuihelper.py | 122 + .../bitbake/lib/bb/ui/crumbs/hobcolor.py | 38 + .../bitbake/lib/bb/ui/crumbs/hobwidget.py | 904 ++++++ .../bitbake/lib/bb/ui/crumbs/persistenttooltip.py | 186 ++ .../bitbake/lib/bb/ui/crumbs/progress.py | 23 + .../bitbake/lib/bb/ui/crumbs/progressbar.py | 59 + .../bitbake/lib/bb/ui/crumbs/puccho.glade | 606 ++++ .../bitbake/lib/bb/ui/crumbs/runningbuild.py | 551 ++++ .../yocto-poky/bitbake/lib/bb/ui/crumbs/utils.py | 34 + .../yocto-poky/bitbake/lib/bb/ui/depexp.py | 333 ++ .../yocto-poky/bitbake/lib/bb/ui/goggle.py | 121 + .../lib/bb/ui/icons/images/images_display.png | Bin 0 -> 6898 bytes .../lib/bb/ui/icons/images/images_hover.png | Bin 0 -> 7051 bytes .../lib/bb/ui/icons/indicators/add-hover.png | Bin 0 -> 1212 bytes .../bitbake/lib/bb/ui/icons/indicators/add.png | Bin 0 -> 1176 bytes .../bitbake/lib/bb/ui/icons/indicators/alert.png | Bin 0 -> 3954 bytes .../lib/bb/ui/icons/indicators/confirmation.png | Bin 0 -> 5789 bytes .../bitbake/lib/bb/ui/icons/indicators/denied.png | Bin 0 -> 3955 bytes .../bitbake/lib/bb/ui/icons/indicators/error.png | Bin 0 -> 6482 bytes .../bitbake/lib/bb/ui/icons/indicators/info.png | Bin 0 -> 3311 bytes .../bitbake/lib/bb/ui/icons/indicators/issues.png | Bin 0 -> 4549 bytes .../bitbake/lib/bb/ui/icons/indicators/refresh.png | Bin 0 -> 5250 bytes .../lib/bb/ui/icons/indicators/remove-hover.png | Bin 0 -> 2809 bytes .../bitbake/lib/bb/ui/icons/indicators/remove.png | Bin 0 -> 1971 bytes .../bitbake/lib/bb/ui/icons/indicators/tick.png | Bin 0 -> 4563 bytes .../bitbake/lib/bb/ui/icons/info/info_display.png | Bin 0 -> 4117 bytes .../bitbake/lib/bb/ui/icons/info/info_hover.png | Bin 0 -> 4167 bytes .../lib/bb/ui/icons/layers/layers_display.png | Bin 0 -> 4840 bytes .../lib/bb/ui/icons/layers/layers_hover.png | Bin 0 -> 5257 bytes .../lib/bb/ui/icons/packages/packages_display.png | Bin 0 -> 7011 bytes .../lib/bb/ui/icons/packages/packages_hover.png | Bin 0 -> 7121 bytes .../lib/bb/ui/icons/recipe/recipe_display.png | Bin 0 -> 4723 bytes .../lib/bb/ui/icons/recipe/recipe_hover.png | Bin 0 -> 4866 bytes .../lib/bb/ui/icons/settings/settings_display.png | Bin 0 -> 6076 bytes .../lib/bb/ui/icons/settings/settings_hover.png | Bin 0 -> 6269 bytes .../bb/ui/icons/templates/templates_display.png | Bin 0 -> 5651 bytes .../lib/bb/ui/icons/templates/templates_hover.png | Bin 0 -> 5791 bytes .../yocto-poky/bitbake/lib/bb/ui/knotty.py | 594 ++++ .../yocto-poky/bitbake/lib/bb/ui/ncurses.py | 373 +++ .../yocto-poky/bitbake/lib/bb/ui/toasterui.py | 465 +++ .../yocto-poky/bitbake/lib/bb/ui/uievent.py | 161 + .../yocto-poky/bitbake/lib/bb/ui/uihelper.py | 59 + import-layers/yocto-poky/bitbake/lib/bb/utils.py | 1453 +++++++++ .../yocto-poky/bitbake/lib/bs4/AUTHORS.txt | 43 + .../yocto-poky/bitbake/lib/bs4/COPYING.txt | 26 + import-layers/yocto-poky/bitbake/lib/bs4/NEWS.txt | 1066 +++++++ .../yocto-poky/bitbake/lib/bs4/__init__.py | 406 +++ .../yocto-poky/bitbake/lib/bs4/builder/__init__.py | 321 ++ .../bitbake/lib/bs4/builder/_html5lib.py | 285 ++ .../bitbake/lib/bs4/builder/_htmlparser.py | 258 ++ .../yocto-poky/bitbake/lib/bs4/builder/_lxml.py | 233 ++ import-layers/yocto-poky/bitbake/lib/bs4/dammit.py | 829 +++++ .../yocto-poky/bitbake/lib/bs4/diagnose.py | 204 ++ .../yocto-poky/bitbake/lib/bs4/element.py | 1611 ++++++++++ .../yocto-poky/bitbake/lib/bs4/testing.py | 592 ++++ .../yocto-poky/bitbake/lib/bs4/tests/__init__.py | 1 + .../bitbake/lib/bs4/tests/test_builder_registry.py | 141 + .../yocto-poky/bitbake/lib/bs4/tests/test_docs.py | 36 + .../bitbake/lib/bs4/tests/test_html5lib.py | 85 + .../bitbake/lib/bs4/tests/test_htmlparser.py | 19 + .../yocto-poky/bitbake/lib/bs4/tests/test_lxml.py | 91 + .../yocto-poky/bitbake/lib/bs4/tests/test_soup.py | 434 +++ .../yocto-poky/bitbake/lib/bs4/tests/test_tree.py | 1829 +++++++++++ import-layers/yocto-poky/bitbake/lib/codegen.py | 570 ++++ .../yocto-poky/bitbake/lib/ply/__init__.py | 4 + import-layers/yocto-poky/bitbake/lib/ply/lex.py | 1058 +++++++ import-layers/yocto-poky/bitbake/lib/ply/yacc.py | 3276 ++++++++++++++++++++ .../yocto-poky/bitbake/lib/progressbar.py | 384 +++ .../yocto-poky/bitbake/lib/prserv/__init__.py | 14 + import-layers/yocto-poky/bitbake/lib/prserv/db.py | 276 ++ .../yocto-poky/bitbake/lib/prserv/serv.py | 471 +++ import-layers/yocto-poky/bitbake/lib/pyinotify.py | 2416 +++++++++++++++ .../yocto-poky/bitbake/lib/toaster/__init__.py | 0 .../bitbake/lib/toaster/bldcollector/__init__.py | 0 .../bitbake/lib/toaster/bldcollector/admin.py | 33 + .../bitbake/lib/toaster/bldcollector/urls.py | 25 + .../bitbake/lib/toaster/bldcollector/views.py | 62 + .../bitbake/lib/toaster/bldcontrol/__init__.py | 0 .../bitbake/lib/toaster/bldcontrol/admin.py | 8 + .../bitbake/lib/toaster/bldcontrol/bbcontroller.py | 142 + .../toaster/bldcontrol/localhostbecontroller.py | 332 ++ .../lib/toaster/bldcontrol/management/__init__.py | 0 .../bldcontrol/management/commands/__init__.py | 0 .../management/commands/checksettings.py | 161 + .../bldcontrol/management/commands/loadconf.py | 183 ++ .../bldcontrol/management/commands/runbuilds.py | 179 ++ .../toaster/bldcontrol/migrations/0001_initial.py | 113 + .../migrations/0002_auto_20160120_1250.py | 19 + .../migrations/0003_add_cancelling_state.py | 19 + .../lib/toaster/bldcontrol/migrations/__init__.py | 0 .../bitbake/lib/toaster/bldcontrol/models.py | 160 + .../bitbake/lib/toaster/bldcontrol/tests.py | 160 + .../bitbake/lib/toaster/bldcontrol/views.py | 1 + .../yocto-poky/bitbake/lib/toaster/contrib/README | 6 + .../bitbake/lib/toaster/contrib/tts/README | 41 + .../bitbake/lib/toaster/contrib/tts/TODO | 9 + .../bitbake/lib/toaster/contrib/tts/config.py | 98 + .../bitbake/lib/toaster/contrib/tts/launcher.py | 101 + .../bitbake/lib/toaster/contrib/tts/recv.py | 56 + .../bitbake/lib/toaster/contrib/tts/runner.py | 222 ++ .../bitbake/lib/toaster/contrib/tts/settings.json | 5 + .../bitbake/lib/toaster/contrib/tts/shellutils.py | 141 + .../bitbake/lib/toaster/contrib/tts/tests.py | 115 + .../contrib/tts/toasteruitest/run_toastertests.py | 155 + .../tts/toasteruitest/toaster_automation_test.py | 2376 ++++++++++++++ .../contrib/tts/toasteruitest/toaster_test.cfg | 25 + .../bitbake/lib/toaster/contrib/tts/urlcheck.py | 53 + .../bitbake/lib/toaster/contrib/tts/urllist.py | 39 + .../yocto-poky/bitbake/lib/toaster/manage.py | 10 + .../yocto-poky/bitbake/lib/toaster/orm/__init__.py | 0 .../bitbake/lib/toaster/orm/management/__init__.py | 0 .../toaster/orm/management/commands/__init__.py | 0 .../toaster/orm/management/commands/lsupdates.py | 12 + .../lib/toaster/orm/migrations/0001_initial.py | 504 +++ .../orm/migrations/0002_customimagerecipe.py | 24 + .../orm/migrations/0003_customimagepackage.py | 24 + .../lib/toaster/orm/migrations/0004_provides.py | 27 + .../orm/migrations/0005_task_field_separation.py | 48 + .../orm/migrations/0006_add_cancelled_state.py | 19 + .../bitbake/lib/toaster/orm/migrations/__init__.py | 0 .../yocto-poky/bitbake/lib/toaster/orm/models.py | 1722 ++++++++++ .../yocto-poky/bitbake/lib/toaster/orm/tests.py | 180 ++ .../bitbake/lib/toaster/tests/__init__.py | 0 .../bitbake/lib/toaster/tests/browser/README | 41 + .../bitbake/lib/toaster/tests/browser/__init__.py | 0 .../lib/toaster/tests/browser/selenium_helpers.py | 204 ++ .../toaster/tests/browser/test_all_builds_page.py | 143 + .../tests/browser/test_all_projects_page.py | 214 ++ .../tests/browser/test_builddashboard_page.py | 251 ++ .../toaster/tests/browser/test_js_unit_tests.py | 57 + .../lib/toaster/tests/browser/test_landing_page.py | 108 + .../tests/browser/test_new_custom_image_page.py | 160 + .../tests/browser/test_project_builds_page.py | 168 + .../lib/toaster/tests/browser/test_project_page.py | 59 + .../lib/toaster/tests/browser/test_sample.py | 41 + .../toaster/tests/toaster-tests-requirements.txt | 1 + .../bitbake/lib/toaster/toastergui/__init__.py | 0 .../bitbake/lib/toaster/toastergui/api.py | 110 + .../fixtures/toastergui-unittest-data.xml | 446 +++ .../static/css/bootstrap-responsive.min.css | 9 + .../toastergui/static/css/bootstrap.min.css | 9 + .../lib/toaster/toastergui/static/css/default.css | 343 ++ .../toastergui/static/css/font-awesome.min.css | 33 + .../ui-bg_diagonals-thick_18_b81900_40x40.png | Bin 0 -> 418 bytes .../ui-bg_diagonals-thick_20_666666_40x40.png | Bin 0 -> 312 bytes .../css/images/ui-bg_flat_10_000000_40x100.png | Bin 0 -> 205 bytes .../css/images/ui-bg_glass_100_f6f6f6_1x400.png | Bin 0 -> 262 bytes .../css/images/ui-bg_glass_100_fdf5ce_1x400.png | Bin 0 -> 348 bytes .../css/images/ui-bg_glass_65_ffffff_1x400.png | Bin 0 -> 207 bytes .../images/ui-bg_gloss-wave_35_f6a828_500x100.png | Bin 0 -> 5815 bytes .../ui-bg_highlight-soft_100_eeeeee_1x100.png | Bin 0 -> 278 bytes .../ui-bg_highlight-soft_75_ffe45c_1x100.png | Bin 0 -> 328 bytes .../static/css/images/ui-icons_222222_256x240.png | Bin 0 -> 6922 bytes .../static/css/images/ui-icons_228ef1_256x240.png | Bin 0 -> 4549 bytes .../static/css/images/ui-icons_ef8c08_256x240.png | Bin 0 -> 4549 bytes .../static/css/images/ui-icons_ffd27a_256x240.png | Bin 0 -> 4549 bytes .../static/css/images/ui-icons_ffffff_256x240.png | Bin 0 -> 6299 bytes .../toastergui/static/css/jquery-ui.min.css | 7 + .../static/css/jquery-ui.structure.min.css | 5 + .../toastergui/static/css/jquery-ui.theme.min.css | 5 + .../toastergui/static/css/jquery.treetable.css | 28 + .../static/css/jquery.treetable.theme.default.css | 64 + .../static/css/jquery.treetable.theme.toaster.css | 38 + .../lib/toaster/toastergui/static/css/prettify.css | 1 + .../toaster/toastergui/static/css/qunit-1.18.0.css | 1 + .../lib/toaster/toastergui/static/css/screen.css | 28 + .../toastergui/static/fonts/FontAwesome.otf | Bin 0 -> 48748 bytes .../static/fonts/fontawesome-webfont.eot | Bin 0 -> 25395 bytes .../static/fonts/fontawesome-webfont.svg | 284 ++ .../static/fonts/fontawesome-webfont.ttf | Bin 0 -> 55096 bytes .../static/fonts/fontawesome-webfont.woff | Bin 0 -> 29380 bytes .../static/fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20290 bytes .../static/fonts/glyphicons-halflings-regular.svg | 229 ++ .../static/fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 41236 bytes .../static/fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23292 bytes .../toastergui/static/html/layer_deps_modal.html | 17 + .../static/img/glyphicons-halflings-white.png | Bin 0 -> 8777 bytes .../toastergui/static/img/glyphicons-halflings.png | Bin 0 -> 12799 bytes .../lib/toaster/toastergui/static/img/logo.png | Bin 0 -> 3020 bytes .../toaster/toastergui/static/img/toaster_bw.png | Bin 0 -> 140514 bytes .../jquery-treetable-license/GPL-LICENSE.txt | 278 ++ .../jquery-treetable-license/MIT-LICENSE.txt | 20 + .../static/jquery-treetable-license/README.md | 20 + .../static/jquery.treetable.theme.toaster.css | 66 + .../lib/toaster/toastergui/static/js/.jshintrc | 11 + .../toaster/toastergui/static/js/bootstrap.min.js | 6 + .../toaster/toastergui/static/js/customrecipe.js | 284 ++ .../toaster/toastergui/static/js/filtersnippet.js | 94 + .../toaster/toastergui/static/js/importlayer.js | 303 ++ .../toastergui/static/js/jquery-2.0.3.min.js | 6 + .../toastergui/static/js/jquery-2.0.3.min.map | 1 + .../toaster/toastergui/static/js/jquery-ui.min.js | 7 + .../toaster/toastergui/static/js/jquery.cookie.js | 117 + .../toastergui/static/js/jquery.treetable.js | 620 ++++ .../lib/toaster/toastergui/static/js/layerBtn.js | 83 + .../toaster/toastergui/static/js/layerDepsModal.js | 90 + .../toaster/toastergui/static/js/layerdetails.js | 395 +++ .../lib/toaster/toastergui/static/js/libtoaster.js | 623 ++++ .../lib/toaster/toastergui/static/js/mrbsection.js | 95 + .../toastergui/static/js/newcustomimage_modal.js | 148 + .../lib/toaster/toastergui/static/js/prettify.js | 28 + .../toaster/toastergui/static/js/projectpage.js | 446 +++ .../toaster/toastergui/static/js/projecttopbar.js | 90 + .../toaster/toastergui/static/js/qunit-1.18.0.js | 347 +++ .../toaster/toastergui/static/js/recipedetails.js | 51 + .../lib/toaster/toastergui/static/js/table.js | 793 +++++ .../lib/toaster/toastergui/static/js/tests/test.js | 175 ++ .../static/js/ui-bootstrap-tpls-0.11.0.js | 10 + .../static/js/ui-bootstrap-tpls-0.11.0.min.js | 10 + .../bitbake/lib/toaster/toastergui/tablefilter.py | 292 ++ .../bitbake/lib/toaster/toastergui/tables.py | 1503 +++++++++ .../lib/toaster/toastergui/templates/base.html | 126 + .../toastergui/templates/basebuilddetailpage.html | 32 + .../toastergui/templates/basebuildpage.html | 157 + .../templates/baseprojectbuildspage.html | 15 + .../toastergui/templates/baseprojectpage.html | 45 + .../toastergui/templates/basetable_bottom.html | 94 + .../toastergui/templates/basetable_top.html | 239 ++ .../toastergui/templates/basetable_top_layers.html | 5 + .../lib/toaster/toastergui/templates/bfile.html | 24 + .../lib/toaster/toastergui/templates/bpackage.html | 108 + .../toaster/toastergui/templates/brtargets.html | 20 + .../toastergui/templates/builddashboard.html | 278 ++ .../toastergui/templates/buildrequestdetails.html | 64 + .../toastergui/templates/builds-toastertable.html | 48 + .../toaster/toastergui/templates/buildtime.html | 4 + .../toastergui/templates/configuration.html | 72 + .../toaster/toastergui/templates/configvars.html | 133 + .../lib/toaster/toastergui/templates/cpuusage.html | 4 + .../toastergui/templates/customise_btn.html | 15 + .../toaster/toastergui/templates/customrecipe.html | 223 ++ .../templates/detail_pagination_bottom.html | 55 + .../toastergui/templates/detail_search_header.html | 68 + .../toastergui/templates/detail_sorted_header.html | 25 + .../lib/toaster/toastergui/templates/dirinfo.html | 241 ++ .../lib/toaster/toastergui/templates/diskio.html | 4 + .../templates/editcustomimage_modal.html | 71 + .../toastergui/templates/filtersnippet.html | 57 + .../templates/generic-toastertable-page.html | 17 + .../toaster/toastergui/templates/importlayer.html | 139 + .../toastergui/templates/js-unit-tests.html | 47 + .../lib/toaster/toastergui/templates/landing.html | 62 + .../toastergui/templates/landing_not_managed.html | 34 + .../toaster/toastergui/templates/layer_btn.html | 17 + .../toaster/toastergui/templates/layerdetails.html | 275 ++ .../toaster/toastergui/templates/machine_btn.html | 16 + .../toaster/toastergui/templates/mrb_section.html | 178 ++ .../toastergui/templates/newcustomimage.html | 14 + .../toastergui/templates/newcustomimage_modal.html | 48 + .../toaster/toastergui/templates/newproject.html | 133 + .../templates/package_built_dependencies.html | 99 + .../toastergui/templates/package_built_detail.html | 65 + .../toastergui/templates/package_detail_base.html | 149 + .../templates/package_included_dependencies.html | 110 + .../templates/package_included_detail.html | 44 + .../package_included_reverse_dependencies.html | 50 + .../templates/package_included_tabs.html | 33 + .../toastergui/templates/pkg_add_rm_btn.html | 34 + .../lib/toaster/toastergui/templates/project.html | 140 + .../templates/projectbuilds-toastertable.html | 56 + .../toastergui/templates/projectbuilds.html | 118 + .../toaster/toastergui/templates/projectconf.html | 950 ++++++ .../templates/projects-toastertable.html | 36 + .../toastergui/templates/projecttopbar.html | 78 + .../lib/toaster/toastergui/templates/recipe.html | 267 ++ .../toaster/toastergui/templates/recipe_btn.html | 16 + .../toastergui/templates/recipe_packages.html | 123 + .../toastergui/templates/recipedetails.html | 177 ++ .../lib/toaster/toastergui/templates/recipes.html | 113 + .../snippets/pkg_dependencies_popover.html | 14 + .../snippets/pkg_revdependencies_popover.html | 14 + .../toaster/toastergui/templates/tablesort.html | 38 + .../lib/toaster/toastergui/templates/target.html | 163 + .../lib/toaster/toastergui/templates/task.html | 277 ++ .../lib/toaster/toastergui/templates/tasks.html | 143 + .../toastergui/templates/toastertable-filter.html | 20 + .../toastergui/templates/toastertable-simple.html | 94 + .../toaster/toastergui/templates/toastertable.html | 105 + .../toastergui/templates/unavailable_artifact.html | 16 + .../toaster/toastergui/templatetags/__init__.py | 0 .../templatetags/objects_to_dictionaries_filter.py | 35 + .../toastergui/templatetags/project_url_tag.py | 34 + .../toaster/toastergui/templatetags/projecttags.py | 299 ++ .../bitbake/lib/toaster/toastergui/tests.py | 494 +++ .../bitbake/lib/toaster/toastergui/typeaheads.py | 146 + .../bitbake/lib/toaster/toastergui/urls.py | 190 ++ .../bitbake/lib/toaster/toastergui/views.py | 2982 ++++++++++++++++++ .../bitbake/lib/toaster/toastergui/widgets.py | 421 +++ .../bitbake/lib/toaster/toastermain/__init__.py | 0 .../lib/toaster/toastermain/management/__init__.py | 0 .../toastermain/management/commands/__init__.py | 0 .../toastermain/management/commands/builddelete.py | 54 + .../toastermain/management/commands/buildslist.py | 13 + .../toastermain/management/commands/checksocket.py | 69 + .../toastermain/management/commands/get-dburl.py | 9 + .../toastermain/management/commands/perf.py | 58 + .../bitbake/lib/toaster/toastermain/settings.py | 396 +++ .../bitbake/lib/toaster/toastermain/urls.py | 90 + .../bitbake/lib/toaster/toastermain/wsgi.py | 35 + .../yocto-poky/bitbake/toaster-requirements.txt | 4 + 437 files changed, 98820 insertions(+) create mode 100644 import-layers/yocto-poky/bitbake/AUTHORS create mode 100644 import-layers/yocto-poky/bitbake/COPYING create mode 100644 import-layers/yocto-poky/bitbake/ChangeLog create mode 100644 import-layers/yocto-poky/bitbake/HEADER create mode 100644 import-layers/yocto-poky/bitbake/LICENSE create mode 100755 import-layers/yocto-poky/bitbake/bin/bitbake create mode 100755 import-layers/yocto-poky/bitbake/bin/bitbake-diffsigs create mode 100755 import-layers/yocto-poky/bitbake/bin/bitbake-dumpsig create mode 100755 import-layers/yocto-poky/bitbake/bin/bitbake-layers create mode 100755 import-layers/yocto-poky/bitbake/bin/bitbake-prserv create mode 100755 import-layers/yocto-poky/bitbake/bin/bitbake-selftest create mode 100755 import-layers/yocto-poky/bitbake/bin/bitbake-worker create mode 100755 import-layers/yocto-poky/bitbake/bin/bitdoc create mode 100755 import-layers/yocto-poky/bitbake/bin/image-writer create mode 100755 import-layers/yocto-poky/bitbake/bin/toaster create mode 100755 import-layers/yocto-poky/bitbake/bin/toaster-eventreplay create mode 100644 import-layers/yocto-poky/bitbake/contrib/README create mode 100644 import-layers/yocto-poky/bitbake/contrib/bbdev.sh create mode 100755 import-layers/yocto-poky/bitbake/contrib/dump_cache.py create mode 100644 import-layers/yocto-poky/bitbake/contrib/vim/ftdetect/bitbake.vim create mode 100644 import-layers/yocto-poky/bitbake/contrib/vim/ftplugin/bitbake.vim create mode 100755 import-layers/yocto-poky/bitbake/contrib/vim/plugin/newbb.vim create mode 100644 import-layers/yocto-poky/bitbake/contrib/vim/syntax/bitbake.vim create mode 100644 import-layers/yocto-poky/bitbake/doc/COPYING.GPL create mode 100644 import-layers/yocto-poky/bitbake/doc/COPYING.MIT create mode 100644 import-layers/yocto-poky/bitbake/doc/Makefile create mode 100644 import-layers/yocto-poky/bitbake/doc/README create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-customization.xsl create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-style.css create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/figures/bitbake-title.png create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/html.css create mode 100644 import-layers/yocto-poky/bitbake/doc/bitbake.1 create mode 100644 import-layers/yocto-poky/bitbake/doc/poky.ent create mode 100644 import-layers/yocto-poky/bitbake/doc/template/Vera.ttf create mode 100644 import-layers/yocto-poky/bitbake/doc/template/Vera.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/template/VeraMoBd.ttf create mode 100644 import-layers/yocto-poky/bitbake/doc/template/VeraMoBd.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/template/VeraMono.ttf create mode 100644 import-layers/yocto-poky/bitbake/doc/template/VeraMono.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/template/component.title.xsl create mode 100644 import-layers/yocto-poky/bitbake/doc/template/db-pdf.xsl create mode 100644 import-layers/yocto-poky/bitbake/doc/template/division.title.xsl create mode 100644 import-layers/yocto-poky/bitbake/doc/template/draft.png create mode 100644 import-layers/yocto-poky/bitbake/doc/template/fop-config.xml create mode 100644 import-layers/yocto-poky/bitbake/doc/template/formal.object.heading.xsl create mode 100644 import-layers/yocto-poky/bitbake/doc/template/gloss-permalinks.xsl create mode 100644 import-layers/yocto-poky/bitbake/doc/template/permalinks.xsl create mode 100644 import-layers/yocto-poky/bitbake/doc/template/section.title.xsl create mode 100644 import-layers/yocto-poky/bitbake/doc/template/titlepage.templates.xml create mode 100755 import-layers/yocto-poky/bitbake/doc/tools/docbook-to-pdf create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/COW.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/build.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/cache.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/cache_extra.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/checksum.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/codeparser.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/command.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/compat.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/cooker.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/daemonize.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/data.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/data_smart.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/event.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/exceptions.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/bzr.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/clearcase.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/cvs.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitannex.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/hg.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/local.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/osc.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/perforce.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/repo.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/sftp.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/ssh.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/svn.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py create mode 100755 import-layers/yocto-poky/bitbake/lib/bb/main.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/methodpool.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/monitordisk.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/msg.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/namedtuple_with_abc.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/persist_data.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/process.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/providers.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/pysh/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/pysh/builtin.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/pysh/interp.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/pysh/lsprof.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/pysh/pysh.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/pysh/pyshlex.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/pysh/pyshyacc.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/pysh/sherrors.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/pysh/subprocess_fix.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/runqueue.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/server/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/server/process.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/shell.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/siggen.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/taskdata.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/tests/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/tests/codeparser.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/tests/cow.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/tests/data.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/tests/parse.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/tests/utils.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/crumbsdialog.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/crumbsmessagedialog.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/deployimagedialog.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/imageselectiondialog.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/layerselectiondialog.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/propertydialog.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/settingsuihelper.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hobcolor.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hobwidget.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/persistenttooltip.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/progress.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/progressbar.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/puccho.glade create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/runningbuild.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/utils.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/depexp.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/goggle.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/images/images_display.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/images/images_hover.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/add-hover.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/add.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/alert.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/confirmation.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/denied.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/error.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/info.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/issues.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/refresh.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/remove-hover.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/remove.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/tick.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/info/info_display.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/info/info_hover.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/layers/layers_display.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/layers/layers_hover.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/packages/packages_display.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/packages/packages_hover.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/recipe/recipe_display.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/recipe/recipe_hover.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/settings/settings_display.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/settings/settings_hover.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/templates/templates_display.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/icons/templates/templates_hover.png create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/knotty.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/ncurses.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/toasterui.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/uievent.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/ui/uihelper.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bb/utils.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/AUTHORS.txt create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/COPYING.txt create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/NEWS.txt create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/builder/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/builder/_html5lib.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/builder/_htmlparser.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/builder/_lxml.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/dammit.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/diagnose.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/element.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/testing.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/tests/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/tests/test_builder_registry.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/tests/test_docs.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/tests/test_html5lib.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/tests/test_htmlparser.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/tests/test_lxml.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/tests/test_soup.py create mode 100644 import-layers/yocto-poky/bitbake/lib/bs4/tests/test_tree.py create mode 100644 import-layers/yocto-poky/bitbake/lib/codegen.py create mode 100644 import-layers/yocto-poky/bitbake/lib/ply/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/ply/lex.py create mode 100644 import-layers/yocto-poky/bitbake/lib/ply/yacc.py create mode 100644 import-layers/yocto-poky/bitbake/lib/progressbar.py create mode 100644 import-layers/yocto-poky/bitbake/lib/prserv/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/prserv/db.py create mode 100644 import-layers/yocto-poky/bitbake/lib/prserv/serv.py create mode 100644 import-layers/yocto-poky/bitbake/lib/pyinotify.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcollector/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcollector/admin.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcollector/urls.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcollector/views.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/admin.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/models.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/views.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/contrib/README create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/README create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/TODO create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/config.py create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/launcher.py create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/recv.py create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/runner.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/settings.json create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/shellutils.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/tests.py create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_automation_test.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_test.cfg create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/urlcheck.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/urllist.py create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/manage.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/management/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/management/commands/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0001_initial.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0002_customimagerecipe.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0003_customimagepackage.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0004_provides.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0006_add_cancelled_state.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/orm/tests.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/README create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/selenium_helpers.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_all_builds_page.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_all_projects_page.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_builddashboard_page.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_js_unit_tests.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_landing_page.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_project_builds_page.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_project_page.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_sample.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/tests/toaster-tests-requirements.txt create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/api.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/bootstrap-responsive.min.css create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/bootstrap.min.css create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/default.css create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/font-awesome.min.css create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_diagonals-thick_18_b81900_40x40.png create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_diagonals-thick_20_666666_40x40.png create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_flat_10_000000_40x100.png create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_glass_100_f6f6f6_1x400.png create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_glass_100_fdf5ce_1x400.png create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_glass_65_ffffff_1x400.png create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_gloss-wave_35_f6a828_500x100.png create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_highlight-soft_100_eeeeee_1x100.png create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_highlight-soft_75_ffe45c_1x100.png create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-icons_222222_256x240.png create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-icons_228ef1_256x240.png create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-icons_ef8c08_256x240.png create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-icons_ffd27a_256x240.png create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/images/ui-icons_ffffff_256x240.png create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/jquery-ui.min.css create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/jquery-ui.structure.min.css create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/jquery-ui.theme.min.css create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/jquery.treetable.css create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/jquery.treetable.theme.default.css create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/jquery.treetable.theme.toaster.css create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/prettify.css create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/qunit-1.18.0.css create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/screen.css create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/fonts/FontAwesome.otf create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/fonts/fontawesome-webfont.eot create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/fonts/fontawesome-webfont.svg create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/fonts/fontawesome-webfont.ttf create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/fonts/fontawesome-webfont.woff create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/fonts/glyphicons-halflings-regular.eot create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/fonts/glyphicons-halflings-regular.svg create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/fonts/glyphicons-halflings-regular.ttf create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/fonts/glyphicons-halflings-regular.woff create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/html/layer_deps_modal.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/img/glyphicons-halflings-white.png create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/img/glyphicons-halflings.png create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/img/logo.png create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/img/toaster_bw.png create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/jquery-treetable-license/GPL-LICENSE.txt create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/jquery-treetable-license/MIT-LICENSE.txt create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/jquery-treetable-license/README.md create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/jquery.treetable.theme.toaster.css create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/.jshintrc create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/bootstrap.min.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/customrecipe.js create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/filtersnippet.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/importlayer.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/jquery-2.0.3.min.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/jquery-2.0.3.min.map create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/jquery-ui.min.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/jquery.cookie.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/jquery.treetable.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/layerBtn.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/layerDepsModal.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/layerdetails.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/libtoaster.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/mrbsection.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/newcustomimage_modal.js create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/prettify.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projectpage.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projecttopbar.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/qunit-1.18.0.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/recipedetails.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/table.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/tests/test.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/ui-bootstrap-tpls-0.11.0.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/ui-bootstrap-tpls-0.11.0.min.js create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/tablefilter.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/tables.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/base.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/basebuilddetailpage.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/basebuildpage.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/baseprojectbuildspage.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/baseprojectpage.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/basetable_bottom.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/basetable_top.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/basetable_top_layers.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/bfile.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/bpackage.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/brtargets.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/builddashboard.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/buildrequestdetails.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/builds-toastertable.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/buildtime.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/configuration.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/configvars.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/cpuusage.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/customise_btn.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/customrecipe.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/detail_pagination_bottom.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/detail_search_header.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/detail_sorted_header.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/dirinfo.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/diskio.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/editcustomimage_modal.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/filtersnippet.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/generic-toastertable-page.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/importlayer.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/js-unit-tests.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/landing.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/landing_not_managed.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/layer_btn.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/layerdetails.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/machine_btn.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/mrb_section.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/newcustomimage.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/newcustomimage_modal.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/newproject.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/package_built_dependencies.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/package_built_detail.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/package_detail_base.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/package_included_dependencies.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/package_included_detail.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/package_included_reverse_dependencies.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/package_included_tabs.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/pkg_add_rm_btn.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/project.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds-toastertable.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectconf.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/projects-toastertable.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/projecttopbar.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/recipe.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/recipe_btn.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/recipe_packages.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/recipedetails.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/recipes.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/snippets/pkg_dependencies_popover.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/snippets/pkg_revdependencies_popover.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/tablesort.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/target.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/task.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/tasks.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/toastertable-filter.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/toastertable-simple.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/toastertable.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/unavailable_artifact.html create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templatetags/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templatetags/objects_to_dictionaries_filter.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templatetags/project_url_tag.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templatetags/projecttags.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/tests.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/typeaheads.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/urls.py create mode 100755 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/views.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastergui/widgets.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/__init__.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/buildslist.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/checksocket.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/get-dburl.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/perf.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/urls.py create mode 100644 import-layers/yocto-poky/bitbake/lib/toaster/toastermain/wsgi.py create mode 100644 import-layers/yocto-poky/bitbake/toaster-requirements.txt (limited to 'import-layers/yocto-poky/bitbake') diff --git a/import-layers/yocto-poky/bitbake/AUTHORS b/import-layers/yocto-poky/bitbake/AUTHORS new file mode 100644 index 000000000..91fd78fd2 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/AUTHORS @@ -0,0 +1,10 @@ +Tim Ansell +Phil Blundell +Seb Frankengul +Holger Freyther +Marcin Juszkiewicz +Chris Larson +Ulrich Luckas +Mickey Lauer +Richard Purdie +Holger Schurig diff --git a/import-layers/yocto-poky/bitbake/COPYING b/import-layers/yocto-poky/bitbake/COPYING new file mode 100644 index 000000000..d511905c1 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/import-layers/yocto-poky/bitbake/ChangeLog b/import-layers/yocto-poky/bitbake/ChangeLog new file mode 100644 index 000000000..4ac2a6446 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/ChangeLog @@ -0,0 +1,317 @@ +Changes in Bitbake 1.9.x: + - Add PE (Package Epoch) support from Philipp Zabel (pH5) + - Treat python functions the same as shell functions for logging + - Use TMPDIR/anonfunc as a __anonfunc temp directory (T) + - Catch truncated cache file errors + - Allow operations other than assignment on flag variables + - Add code to handle inter-task dependencies + - Fix cache errors when generation dotGraphs + - Make sure __inherit_cache is updated before calling include() (from Michael Krelin) + - Fix bug when target was in ASSUME_PROVIDED (#2236) + - Raise ParseError for filenames with multiple underscores instead of infinitely looping (#2062) + - Fix invalid regexp in BBMASK error handling (missing import) (#1124) + - Promote certain warnings from debug to note 2 level + - Update manual + - Correctly redirect stdin when forking + - If parsing errors are found, exit, too many users miss the errors + - Remove supriours PREFERRED_PROVIDER warnings + - svn fetcher: Add _buildsvncommand function + - Improve certain error messages + - Rewrite svn fetcher to make adding extra operations easier + as part of future SRCDATE="now" fixes + (requires new FETCHCMD_svn definition in bitbake.conf) + - Change SVNDIR layout to be more unique (fixes #2644 and #2624) + - Add ConfigParsed Event after configuration parsing is complete + - Add SRCREV support for svn fetcher + - data.emit_var() - only call getVar if we need the variable + - Stop generating the A variable (seems to be legacy code) + - Make sure intertask depends get processed correcting in recursive depends + - Add pn-PN to overrides when evaluating PREFERRED_VERSION + - Improve the progress indicator by skipping tasks that have + already run before starting the build rather than during it + - Add profiling option (-P) + - Add BB_SRCREV_POLICY variable (clear or cache) to control SRCREV cache + - Add SRCREV_FORMAT support + - Fix local fetcher's localpath return values + - Apply OVERRIDES before performing immediate expansions + - Allow the -b -e option combination to take regular expressions + - Fix handling of variables with expansion in the name using _append/_prepend + e.g. RRECOMMENDS_${PN}_append_xyz = "abc" + - Add plain message function to bb.msg + - Sort the list of providers before processing so dependency problems are + reproducible rather than effectively random + - Fix/improve bitbake -s output + - Add locking for fetchers so only one tries to fetch a given file at a given time + - Fix int(0)/None confusion in runqueue.py which causes random gaps in dependency chains + - Expand data in addtasks + - Print the list of missing DEPENDS,RDEPENDS for the "No buildable providers available for required...." + error message. + - Rework add_task to be more efficient (6% speedup, 7% number of function calls reduction) + - Sort digraph output to make builds more reproducible + - Split expandKeys into two for loops to benefit from the expand_cache (12% speedup) + - runqueue.py: Fix idepends handling to avoid dependency errors + - Clear the terminal TOSTOP flag if set (and warn the user) + - Fix regression from r653 and make SRCDATE/CVSDATE work for packages again + - Fix a bug in bb.decodeurl where http://some.where.com/somefile.tgz decoded to host="" (#1530) + - Warn about malformed PREFERRED_PROVIDERS (#1072) + - Add support for BB_NICE_LEVEL option (#1627) + - Psyco is used only on x86 as there is no support for other architectures. + - Sort initial providers list by default preference (#1145, #2024) + - Improve provider sorting so prefered versions have preference over latest versions (#768) + - Detect builds of tasks with overlapping providers and warn (will become a fatal error) (#1359) + - Add MULTI_PROVIDER_WHITELIST variable to allow known safe multiple providers to be listed + - Handle paths in svn fetcher module parameter + - Support the syntax "export VARIABLE" + - Add bzr fetcher + - Add support for cleaning directories before a task in the form: + do_taskname[cleandirs] = "dir" + - bzr fetcher tweaks from Robert Schuster (#2913) + - Add mercurial (hg) fetcher from Robert Schuster (#2913) + - Don't add duplicates to BBPATH + - Fix preferred_version return values (providers.py) + - Fix 'depends' flag splitting + - Fix unexport handling (#3135) + - Add bb.copyfile function similar to bb.movefile (and improve movefile error reporting) + - Allow multiple options for deptask flag + - Use git-fetch instead of git-pull removing any need for merges when + fetching (we don't care about the index). Fixes fetch errors. + - Add BB_GENERATE_MIRROR_TARBALLS option, set to 0 to make git fetches + faster at the expense of not creating mirror tarballs. + - SRCREV handling updates, improvements and fixes from Poky + - Add bb.utils.lockfile() and bb.utils.unlockfile() from Poky + - Add support for task selfstamp and lockfiles flags + - Disable task number acceleration since it can allow the tasks to run + out of sequence + - Improve runqueue code comments + - Add task scheduler abstraction and some example schedulers + - Improve circular dependency chain debugging code and user feedback + - Don't give a stacktrace for invalid tasks, have a user friendly message (#3431) + - Add support for "-e target" (#3432) + - Fix shell showdata command (#3259) + - Fix shell data updating problems (#1880) + - Properly raise errors for invalid source URI protocols + - Change the wget fetcher failure handling to avoid lockfile problems + - Add support for branches in git fetcher (Otavio Salvador, Michael Lauer) + - Make taskdata and runqueue errors more user friendly + - Add norecurse and fullpath options to cvs fetcher + - Fix exit code for build failures in --continue mode + - Fix git branch tags fetching + - Change parseConfigurationFile so it works on real data, not a copy + - Handle 'base' inherit and all other INHERITs from parseConfigurationFile + instead of BBHandler + - Fix getVarFlags bug in data_smart + - Optmise cache handling by more quickly detecting an invalid cache, only + saving the cache when its changed, moving the cache validity check into + the parsing loop and factoring some getVar calls outside a for loop + - Cooker: Remove a debug message from the parsing loop to lower overhead + - Convert build.py exec_task to use getVarFlags + - Update shell to use cooker.buildFile + - Add StampUpdate event + - Convert -b option to use taskdata/runqueue + - Remove digraph and switch to new stamp checking code. exec_task no longer + honours dependencies + - Make fetcher timestamp updating non-fatal when permissions don't allow + updates + - Add BB_SCHEDULER variable/option ("completion" or "speed") controlling + the way bitbake schedules tasks + - Add BB_STAMP_POLICY variable/option ("perfile" or "full") controlling + how extensively stamps are looked at for validity + - When handling build target failures make sure idepends are checked and + failed where needed. Fixes --continue mode crashes. + - Fix -f (force) in conjunction with -b + - Fix problems with recrdeptask handling where some idepends weren't handled + correctly. + - Handle exit codes correctly (from pH5) + - Work around refs/HEAD issues with git over http (#3410) + - Add proxy support to the CVS fetcher (from Cyril Chemparathy) + - Improve runfetchcmd so errors are seen and various GIT variables are exported + - Add ability to fetchers to check URL validity without downloading + - Improve runtime PREFERRED_PROVIDERS warning message + - Add BB_STAMP_WHITELIST option which contains a list of stamps to ignore when + checking stamp dependencies and using a BB_STAMP_POLICY of "whitelist" + - No longer weight providers on the basis of a package being "already staged". This + leads to builds being non-deterministic. + - Flush stdout/stderr before forking to fix duplicate console output + - Make sure recrdeps tasks include all inter-task dependencies of a given fn + - Add bb.runqueue.check_stamp_fn() for use by packaged-staging + - Add PERSISTENT_DIR to store the PersistData in a persistent + directory != the cache dir. + - Add md5 and sha256 checksum generation functions to utils.py + - Correctly handle '-' characters in class names (#2958) + - Make sure expandKeys has been called on the data dictionary before running tasks + - Correctly add a task override in the form task-TASKNAME. + - Revert the '-' character fix in class names since it breaks things + - When a regexp fails to compile for PACKAGES_DYNAMIC, print a more useful error (#4444) + - Allow to checkout CVS by Date and Time. Just add HHmm to the SRCDATE. + - Move prunedir function to utils.py and add explode_dep_versions function + - Raise an exception if SRCREV == 'INVALID' + - Fix hg fetcher username/password handling and fix crash + - Fix PACKAGES_DYNAMIC handling of packages with '++' in the name + - Rename __depends to __base_depends after configuration parsing so we don't + recheck the validity of the config files time after time + - Add better environmental variable handling. By default it will now only pass certain + whitelisted variables into the data store. If BB_PRESERVE_ENV is set bitbake will use + all variable from the environment. If BB_ENV_WHITELIST is set, that whitelist will be + used instead of the internal bitbake one. Alternatively, BB_ENV_EXTRAWHITE can be used + to extend the internal whitelist. + - Perforce fetcher fix to use commandline options instead of being overriden by the environment + - bb.utils.prunedir can cope with symlinks to directoriees without exceptions + - use @rev when doing a svn checkout + - Add osc fetcher (from Joshua Lock in Poky) + - When SRCREV autorevisioning for a recipe is in use, don't cache the recipe + - Add tryaltconfigs option to control whether bitbake trys using alternative providers + to fulfil failed dependencies. It defaults to off, changing the default since this + behaviour confuses many users and isn't often useful. + - Improve lock file function error handling + - Add username handling to the git fetcher (Robert Bragg) + - Add support for HTTP_PROXY and HTTP_PROXY_IGNORE variables to the wget fetcher + - Export more variables to the fetcher commands to allow ssh checkouts and checkouts through + proxies to work better. (from Poky) + - Also allow user and pswd options in SRC_URIs globally (from Poky) + - Improve proxy handling when using mirrors (from Poky) + - Add bb.utils.prune_suffix function + - Fix hg checkouts of specific revisions (from Poky) + - Fix wget fetching of urls with parameters specified (from Poky) + - Add username handling to git fetcher (from Poky) + - Set HOME environmental variable when running fetcher commands (from Poky) + - Make sure allowed variables inherited from the environment are exported again (from Poky) + - When running a stage task in bbshell, run populate_staging, not the stage task (from Poky) + - Fix + character escaping from PACKAGES_DYNAMIC (thanks Otavio Salvador) + - Addition of BBCLASSEXTEND support for allowing one recipe to provide multiple targets (from Poky) + +Changes in Bitbake 1.8.0: + - Release 1.7.x as a stable series + +Changes in BitBake 1.7.x: + - Major updates of the dependency handling and execution + of tasks. Code from bin/bitbake replaced with runqueue.py + and taskdata.py + - New task execution code supports multithreading with a simplistic + threading algorithm controlled by BB_NUMBER_THREADS + - Change of the SVN Fetcher to keep the checkout around + courtsey of Paul Sokolovsky (#1367) + - PATH fix to bbimage (#1108) + - Allow debug domains to be specified on the commandline (-l) + - Allow 'interactive' tasks + - Logging message improvements + - Drop now uneeded BUILD_ALL_DEPS variable + - Add support for wildcards to -b option + - Major overhaul of the fetchers making a large amount of code common + including mirroring code + - Fetchers now touch md5 stamps upon access (to show activity) + - Fix -f force option when used without -b (long standing bug) + - Add expand_cache to data_cache.py, caching expanded data (speedup) + - Allow version field in DEPENDS (ignored for now) + - Add abort flag support to the shell + - Make inherit fail if the class doesn't exist (#1478) + - Fix data.emit_env() to expand keynames as well as values + - Add ssh fetcher + - Add perforce fetcher + - Make PREFERRED_PROVIDER_foobar defaults to foobar if available + - Share the parser's mtime_cache, reducing the number of stat syscalls + - Compile all anonfuncs at once! + *** Anonfuncs must now use common spacing format *** + - Memorise the list of handlers in __BBHANDLERS and tasks in __BBTASKS + This removes 2 million function calls resulting in a 5-10% speedup + - Add manpage + - Update generateDotGraph to use taskData/runQueue improving accuracy + and also adding a task dependency graph + - Fix/standardise on GPLv2 licence + - Move most functionality from bin/bitbake to cooker.py and split into + separate funcitons + - CVS fetcher: Added support for non-default port + - Add BBINCLUDELOGS_LINES, the number of lines to read from any logfile + - Drop shebangs from lib/bb scripts + +Changes in Bitbake 1.6.0: + - Better msg handling + - COW dict implementation from Tim Ansell (mithro) leading + to better performance + - Speed up of -s + +Changes in Bitbake 1.4.4: + - SRCDATE now handling courtsey Justin Patrin + - #1017 fix to work with rm_work + +Changes in BitBake 1.4.2: + - Send logs to oe.pastebin.com instead of pastebin.com + fixes #856 + - Copy the internal bitbake data before building the + dependency graph. This fixes nano not having a + virtual/libc dependency + - Allow multiple TARBALL_STASH entries + - Cache, check if the directory exists before changing + into it + - git speedup cloning by not doing a checkout + - allow to have spaces in filenames (.conf, .bb, .bbclass) + +Changes in BitBake 1.4.0: + - Fix to check both RDEPENDS and RDEPENDS_${PN} + - Fix a RDEPENDS parsing bug in utils:explode_deps() + - Update git fetcher behaviour to match git changes + - ASSUME_PROVIDED allowed to include runtime packages + - git fetcher cleanup and efficency improvements + - Change the format of the cache + - Update usermanual to document the Fetchers + - Major changes to caching with a new strategy + giving a major performance increase when reparsing + with few data changes + +Changes in BitBake 1.3.3: + - Create a new Fetcher module to ease the + development of new Fetchers. + Issue #438 fixed by rpurdie@openedhand.com + - Make the Subversion fetcher honor the SRC Date + (CVSDATE). + Issue #555 fixed by chris@openedhand.com + - Expand PREFERRED_PROVIDER properly + Issue #436 fixed by rprudie@openedhand.com + - Typo fix for Issue #531 by Philipp Zabel for the + BitBake Shell + - Introduce a new special variable SRCDATE as + a generic naming to replace CVSDATE. + - Introduce a new keyword 'required'. In contrast + to 'include' parsing will fail if a to be included + file can not be found. + - Remove hardcoding of the STAMP directory. Patch + courtsey pHilipp Zabel + - Track the RDEPENDS of each package (rpurdie@openedhand.com) + - Introduce BUILD_ALL_DEPS to build all RDEPENDS. E.g + this is used by the OpenEmbedded Meta Packages. + (rpurdie@openedhand.com). + +Changes in BitBake 1.3.2: + - reintegration of make.py into BitBake + - bbread is gone, use bitbake -e + - lots of shell updates and bugfixes + - Introduction of the .= and =. operator + - Sort variables, keys and groups in bitdoc + - Fix regression in the handling of BBCOLLECTIONS + - Update the bitbake usermanual + +Changes in BitBake 1.3.0: + - add bitbake interactive shell (bitbake -i) + - refactor bitbake utility in OO style + - kill default arguments in methods in the bb.data module + - kill default arguments in methods in the bb.fetch module + - the http/https/ftp fetcher will fail if the to be + downloaded file was not found in DL_DIR (this is needed + to avoid unpacking the sourceforge mirror page) + - Switch to a cow like data instance for persistent and non + persisting mode (called data_smart.py) + - Changed the callback of bb.make.collect_bbfiles to carry + additional parameters + - Drastically reduced the amount of needed RAM by not holding + each data instance in memory when using a cache/persistent + storage + +Changes in BitBake 1.2.1: + The 1.2.1 release is meant as a intermediate release to lay the + ground for more radical changes. The most notable changes are: + + - Do not hardcode {}, use bb.data.init() instead if you want to + get a instance of a data class + - bb.data.init() is a factory and the old bb.data methods are delegates + - Do not use deepcopy use bb.data.createCopy() instead. + - Removed default arguments in bb.fetch + diff --git a/import-layers/yocto-poky/bitbake/HEADER b/import-layers/yocto-poky/bitbake/HEADER new file mode 100644 index 000000000..9859255df --- /dev/null +++ b/import-layers/yocto-poky/bitbake/HEADER @@ -0,0 +1,19 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# +# Copyright (C) +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + diff --git a/import-layers/yocto-poky/bitbake/LICENSE b/import-layers/yocto-poky/bitbake/LICENSE new file mode 100644 index 000000000..a57f9a419 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/LICENSE @@ -0,0 +1,12 @@ +BitBake is licensed under the GNU General Public License version 2.0. See COPYING for further details. + +The following external components are distributed with this software: + +* The Toaster Simple UI application is based upon the Django project template, the files of which are covered by the BSD license and are copyright (c) Django Software +Foundation and individual contributors. + +* Twitter Bootstrap (including Glyphicons), redistributed under the Apache License 2.0. + +* jQuery is redistributed under the MIT license. + +* QUnit is redistributed under the MIT license. diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake b/import-layers/yocto-poky/bitbake/bin/bitbake new file mode 100755 index 000000000..b03683e12 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/bitbake @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH +# Copyright (C) 2006 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), + 'lib')) +try: + import bb +except RuntimeError as exc: + sys.exit(str(exc)) + +from bb import cookerdata +from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException + +__version__ = "1.30.0" + +if __name__ == "__main__": + if __version__ != bb.__version__: + sys.exit("Bitbake core version and program version mismatch!") + try: + sys.exit(bitbake_main(BitBakeConfigParameters(sys.argv), + cookerdata.CookerConfiguration())) + except BBMainException as err: + sys.exit(err) + except bb.BBHandledException: + sys.exit(1) + except Exception: + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake-diffsigs b/import-layers/yocto-poky/bitbake/bin/bitbake-diffsigs new file mode 100755 index 000000000..196f0b73e --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/bitbake-diffsigs @@ -0,0 +1,138 @@ +#!/usr/bin/env python + +# bitbake-diffsigs +# BitBake task signature data comparison utility +# +# Copyright (C) 2012-2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys +import warnings +import fnmatch +import optparse +import logging + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) + +import bb.tinfoil +import bb.siggen + +def logger_create(name, output=sys.stderr): + logger = logging.getLogger(name) + console = logging.StreamHandler(output) + format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") + if output.isatty(): + format.enable_color() + console.setFormatter(format) + logger.addHandler(console) + logger.setLevel(logging.INFO) + return logger + +logger = logger_create('bitbake-diffsigs') + +def find_compare_task(bbhandler, pn, taskname): + """ Find the most recent signature files for the specified PN/task and compare them """ + + def get_hashval(siginfo): + if siginfo.endswith('.siginfo'): + return siginfo.rpartition(':')[2].partition('_')[0] + else: + return siginfo.rpartition('.')[2] + + if not hasattr(bb.siggen, 'find_siginfo'): + logger.error('Metadata does not support finding signature data files') + sys.exit(1) + + if not taskname.startswith('do_'): + taskname = 'do_%s' % taskname + + filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data) + latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-3:] + if not latestfiles: + logger.error('No sigdata files found matching %s %s' % (pn, taskname)) + sys.exit(1) + elif len(latestfiles) < 2: + logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname)) + sys.exit(1) + else: + # It's possible that latestfiles contain 3 elements and the first two have the same hash value. + # In this case, we delete the second element. + # The above case is actually the most common one. Because we may have sigdata file and siginfo + # file having the same hash value. Comparing such two files makes no sense. + if len(latestfiles) == 3: + hash0 = get_hashval(latestfiles[0]) + hash1 = get_hashval(latestfiles[1]) + if hash0 == hash1: + latestfiles.pop(1) + + # Define recursion callback + def recursecb(key, hash1, hash2): + hashes = [hash1, hash2] + hashfiles = bb.siggen.find_siginfo(key, None, hashes, bbhandler.config_data) + + recout = [] + if len(hashfiles) == 2: + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb) + recout.extend(list(' ' + l for l in out2)) + else: + recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2)) + + return recout + + # Recurse into signature comparison + output = bb.siggen.compare_sigfiles(latestfiles[0], latestfiles[1], recursecb) + if output: + print '\n'.join(output) + sys.exit(0) + + + +parser = optparse.OptionParser( + description = "Compares siginfo/sigdata files written out by BitBake", + usage = """ + %prog -t recipename taskname + %prog sigdatafile1 sigdatafile2 + %prog sigdatafile1""") + +parser.add_option("-t", "--task", + help = "find the signature data files for last two runs of the specified task and compare them", + action="store", dest="taskargs", nargs=2, metavar='recipename taskname') + +options, args = parser.parse_args(sys.argv) + +if options.taskargs: + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(config_only = True) + find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1]) +else: + if len(args) == 1: + parser.print_help() + else: + import cPickle + try: + if len(args) == 2: + output = bb.siggen.dump_sigfile(sys.argv[1]) + else: + output = bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2]) + except IOError as e: + logger.error(str(e)) + sys.exit(1) + except cPickle.UnpicklingError, EOFError: + logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files') + sys.exit(1) + + if output: + print '\n'.join(output) diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake-dumpsig b/import-layers/yocto-poky/bitbake/bin/bitbake-dumpsig new file mode 100755 index 000000000..656d93a5a --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/bitbake-dumpsig @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# bitbake-dumpsig +# BitBake task signature dump utility +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys +import warnings +import optparse +import logging + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) + +import bb.siggen + +def logger_create(name, output=sys.stderr): + logger = logging.getLogger(name) + console = logging.StreamHandler(output) + format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") + if output.isatty(): + format.enable_color() + console.setFormatter(format) + logger.addHandler(console) + logger.setLevel(logging.INFO) + return logger + +logger = logger_create('bitbake-dumpsig') + +parser = optparse.OptionParser( + description = "Dumps siginfo/sigdata files written out by BitBake", + usage = """ + %prog sigdatafile""") + +options, args = parser.parse_args(sys.argv) + +if len(args) == 1: + parser.print_help() +else: + import cPickle + try: + output = bb.siggen.dump_sigfile(args[1]) + except IOError as e: + logger.error(str(e)) + sys.exit(1) + except cPickle.UnpicklingError, EOFError: + logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file') + sys.exit(1) + + if output: + print '\n'.join(output) diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake-layers b/import-layers/yocto-poky/bitbake/bin/bitbake-layers new file mode 100755 index 000000000..d47a6690e --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/bitbake-layers @@ -0,0 +1,1072 @@ +#!/usr/bin/env python + +# This script has subcommands which operate against your bitbake layers, either +# displaying useful information, or acting against them. +# See the help output for details on available commands. + +# Copyright (C) 2011 Mentor Graphics Corporation +# Copyright (C) 2011-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import logging +import os +import sys +import fnmatch +from collections import defaultdict +import argparse +import re +import httplib, urlparse, json +import subprocess + +bindir = os.path.dirname(__file__) +topdir = os.path.dirname(bindir) +sys.path[0:0] = [os.path.join(topdir, 'lib')] + +import bb.cache +import bb.cooker +import bb.providers +import bb.utils +import bb.tinfoil + + +def logger_create(name, output=sys.stderr): + logger = logging.getLogger(name) + console = logging.StreamHandler(output) + format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") + if output.isatty(): + format.enable_color() + console.setFormatter(format) + logger.addHandler(console) + logger.setLevel(logging.INFO) + return logger + +logger = logger_create('bitbake-layers', sys.stdout) + +class UserError(Exception): + pass + +class Commands(): + def __init__(self): + self.bbhandler = None + self.bblayers = [] + + def init_bbhandler(self, config_only = False): + if not self.bbhandler: + self.bbhandler = bb.tinfoil.Tinfoil(tracking=True) + self.bblayers = (self.bbhandler.config_data.getVar('BBLAYERS', True) or "").split() + self.bbhandler.prepare(config_only) + layerconfs = self.bbhandler.config_data.varhistory.get_variable_items_files('BBFILE_COLLECTIONS', self.bbhandler.config_data) + self.bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.iteritems()} + + + def do_show_layers(self, args): + """show current configured layers""" + self.init_bbhandler(config_only = True) + logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(40), "priority")) + logger.plain('=' * 74) + for layer, _, regex, pri in self.bbhandler.cooker.recipecache.bbfile_config_priorities: + layerdir = self.bbfile_collections.get(layer, None) + layername = self.get_layer_name(layerdir) + logger.plain("%s %s %d" % (layername.ljust(20), layerdir.ljust(40), pri)) + + + def do_add_layer(self, args): + """Add a layer to bblayers.conf + +Adds the specified layer to bblayers.conf +""" + layerdir = os.path.abspath(args.layerdir) + if not os.path.exists(layerdir): + sys.stderr.write("Specified layer directory doesn't exist\n") + return 1 + + layer_conf = os.path.join(layerdir, 'conf', 'layer.conf') + if not os.path.exists(layer_conf): + sys.stderr.write("Specified layer directory doesn't contain a conf/layer.conf file\n") + return 1 + + bblayers_conf = os.path.join('conf', 'bblayers.conf') + if not os.path.exists(bblayers_conf): + sys.stderr.write("Unable to find bblayers.conf\n") + return 1 + + (notadded, _) = bb.utils.edit_bblayers_conf(bblayers_conf, layerdir, None) + if notadded: + for item in notadded: + sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item) + + + def do_remove_layer(self, args): + """Remove a layer from bblayers.conf + +Removes the specified layer from bblayers.conf +""" + bblayers_conf = os.path.join('conf', 'bblayers.conf') + if not os.path.exists(bblayers_conf): + sys.stderr.write("Unable to find bblayers.conf\n") + return 1 + + if args.layerdir.startswith('*'): + layerdir = args.layerdir + elif not '/' in args.layerdir: + layerdir = '*/%s' % args.layerdir + else: + layerdir = os.path.abspath(args.layerdir) + (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdir) + if notremoved: + for item in notremoved: + sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item) + return 1 + + + def get_json_data(self, apiurl): + proxy_settings = os.environ.get("http_proxy", None) + conn = None + _parsedurl = urlparse.urlparse(apiurl) + path = _parsedurl.path + query = _parsedurl.query + def parse_url(url): + parsedurl = urlparse.urlparse(url) + if parsedurl.netloc[0] == '[': + host, port = parsedurl.netloc[1:].split(']', 1) + if ':' in port: + port = port.rsplit(':', 1)[1] + else: + port = None + else: + if parsedurl.netloc.count(':') == 1: + (host, port) = parsedurl.netloc.split(":") + else: + host = parsedurl.netloc + port = None + return (host, 80 if port is None else int(port)) + + if proxy_settings is None: + host, port = parse_url(apiurl) + conn = httplib.HTTPConnection(host, port) + conn.request("GET", path + "?" + query) + else: + host, port = parse_url(proxy_settings) + conn = httplib.HTTPConnection(host, port) + conn.request("GET", apiurl) + + r = conn.getresponse() + if r.status != 200: + raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason)) + return json.loads(r.read()) + + + def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False): + def layeritems_info_id(items_name, layeritems): + litems_id = None + for li in layeritems: + if li['name'] == items_name: + litems_id = li['id'] + break + return litems_id + + def layerbranches_info(items_id, layerbranches): + lbranch = {} + for lb in layerbranches: + if lb['layer'] == items_id and lb['branch'] == branchnum: + lbranch['id'] = lb['id'] + lbranch['vcs_subdir'] = lb['vcs_subdir'] + break + return lbranch + + def layerdependencies_info(lb_id, layerdependencies): + ld_deps = [] + for ld in layerdependencies: + if ld['layerbranch'] == lb_id and not ld['dependency'] in ld_deps: + ld_deps.append(ld['dependency']) + if not ld_deps: + logger.error("The dependency of layerDependencies is not found.") + return ld_deps + + def layeritems_info_name_subdir(items_id, layeritems): + litems = {} + for li in layeritems: + if li['id'] == items_id: + litems['vcs_url'] = li['vcs_url'] + litems['name'] = li['name'] + break + return litems + + if selfname: + selfid = layeritems_info_id(layername, layeritems) + lbinfo = layerbranches_info(selfid, layerbranches) + if lbinfo: + selfsubdir = lbinfo['vcs_subdir'] + else: + logger.error("%s is not found in the specified branch" % layername) + return + selfurl = layeritems_info_name_subdir(selfid, layeritems)['vcs_url'] + if selfurl: + return selfurl, selfsubdir + else: + logger.error("Cannot get layer %s git repo and subdir" % layername) + return + ldict = {} + itemsid = layeritems_info_id(layername, layeritems) + if not itemsid: + return layername, None + lbid = layerbranches_info(itemsid, layerbranches) + if lbid: + lbid = layerbranches_info(itemsid, layerbranches)['id'] + else: + logger.error("%s is not found in the specified branch" % layername) + return None, None + for dependency in layerdependencies_info(lbid, layerdependencies): + lname = layeritems_info_name_subdir(dependency, layeritems)['name'] + lurl = layeritems_info_name_subdir(dependency, layeritems)['vcs_url'] + lsubdir = layerbranches_info(dependency, layerbranches)['vcs_subdir'] + ldict[lname] = lurl, lsubdir + return None, ldict + + + def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer): + layername = self.get_layer_name(url) + if os.path.splitext(layername)[1] == '.git': + layername = os.path.splitext(layername)[0] + repodir = os.path.join(fetchdir, layername) + layerdir = os.path.join(repodir, subdir) + if not os.path.exists(repodir): + if fetch_layer: + result = subprocess.call('git clone %s %s' % (url, repodir), shell = True) + if result: + logger.error("Failed to download %s" % url) + return None, None + else: + return layername, layerdir + else: + logger.plain("Repository %s needs to be fetched" % url) + return layername, layerdir + elif os.path.exists(layerdir): + return layername, layerdir + else: + logger.error("%s is not in %s" % (url, subdir)) + return None, None + + + def do_layerindex_fetch(self, args): + """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf. +""" + self.init_bbhandler(config_only = True) + apiurl = self.bbhandler.config_data.getVar('BBLAYERS_LAYERINDEX_URL', True) + if not apiurl: + logger.error("Cannot get BBLAYERS_LAYERINDEX_URL") + return 1 + else: + if apiurl[-1] != '/': + apiurl += '/' + apiurl += "api/" + apilinks = self.get_json_data(apiurl) + branches = self.get_json_data(apilinks['branches']) + + branchnum = 0 + for branch in branches: + if branch['name'] == args.branch: + branchnum = branch['id'] + break + if branchnum == 0: + validbranches = ', '.join([branch['name'] for branch in branches]) + logger.error('Invalid layer branch name "%s". Valid branches: %s' % (args.branch, validbranches)) + return 1 + + ignore_layers = [] + for collection in self.bbhandler.config_data.getVar('BBFILE_COLLECTIONS', True).split(): + lname = self.bbhandler.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection, True) + if lname: + ignore_layers.append(lname) + + if args.ignore: + ignore_layers.extend(args.ignore.split(',')) + + layeritems = self.get_json_data(apilinks['layerItems']) + layerbranches = self.get_json_data(apilinks['layerBranches']) + layerdependencies = self.get_json_data(apilinks['layerDependencies']) + invaluenames = [] + repourls = {} + printlayers = [] + def query_dependencies(layers, layeritems, layerbranches, layerdependencies, branchnum): + depslayer = [] + for layername in layers: + invaluename, layerdict = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum) + if layerdict: + repourls[layername] = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=True) + for layer in layerdict: + if not layer in ignore_layers: + depslayer.append(layer) + printlayers.append((layername, layer, layerdict[layer][0], layerdict[layer][1])) + if not layer in ignore_layers and not layer in repourls: + repourls[layer] = (layerdict[layer][0], layerdict[layer][1]) + if invaluename and not invaluename in invaluenames: + invaluenames.append(invaluename) + return depslayer + + depslayers = query_dependencies(args.layername, layeritems, layerbranches, layerdependencies, branchnum) + while depslayers: + depslayer = query_dependencies(depslayers, layeritems, layerbranches, layerdependencies, branchnum) + depslayers = depslayer + if invaluenames: + for invaluename in invaluenames: + logger.error('Layer "%s" not found in layer index' % invaluename) + return 1 + logger.plain("%s %s %s %s" % ("Layer".ljust(19), "Required by".ljust(19), "Git repository".ljust(54), "Subdirectory")) + logger.plain('=' * 115) + for layername in args.layername: + layerurl = repourls[layername] + logger.plain("%s %s %s %s" % (layername.ljust(20), '-'.ljust(20), layerurl[0].ljust(55), layerurl[1])) + printedlayers = [] + for layer, dependency, gitrepo, subdirectory in printlayers: + if dependency in printedlayers: + continue + logger.plain("%s %s %s %s" % (dependency.ljust(20), layer.ljust(20), gitrepo.ljust(55), subdirectory)) + printedlayers.append(dependency) + + if repourls: + fetchdir = self.bbhandler.config_data.getVar('BBLAYERS_FETCH_DIR', True) + if not fetchdir: + logger.error("Cannot get BBLAYERS_FETCH_DIR") + return 1 + if not os.path.exists(fetchdir): + os.makedirs(fetchdir) + addlayers = [] + for repourl, subdir in repourls.values(): + name, layerdir = self.get_fetch_layer(fetchdir, repourl, subdir, not args.show_only) + if not name: + # Error already shown + return 1 + addlayers.append((subdir, name, layerdir)) + if not args.show_only: + for subdir, name, layerdir in set(addlayers): + if os.path.exists(layerdir): + if subdir: + logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % subdir) + else: + logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % name) + localargs = argparse.Namespace() + localargs.layerdir = layerdir + self.do_add_layer(localargs) + else: + break + + + def do_layerindex_show_depends(self, args): + """Find layer dependencies from layer index. +""" + args.show_only = True + args.ignore = [] + self.do_layerindex_fetch(args) + + + def version_str(self, pe, pv, pr = None): + verstr = "%s" % pv + if pr: + verstr = "%s-%s" % (verstr, pr) + if pe: + verstr = "%s:%s" % (pe, verstr) + return verstr + + + def do_show_overlayed(self, args): + """list overlayed recipes (where the same recipe exists in another layer) + +Lists the names of overlayed recipes and the available versions in each +layer, with the preferred version first. Note that skipped recipes that +are overlayed will also be listed, with a " (skipped)" suffix. +""" + self.init_bbhandler() + + items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, True, None) + + # Check for overlayed .bbclass files + classes = defaultdict(list) + for layerdir in self.bblayers: + classdir = os.path.join(layerdir, 'classes') + if os.path.exists(classdir): + for classfile in os.listdir(classdir): + if os.path.splitext(classfile)[1] == '.bbclass': + classes[classfile].append(classdir) + + # Locating classes and other files is a bit more complicated than recipes - + # layer priority is not a factor; instead BitBake uses the first matching + # file in BBPATH, which is manipulated directly by each layer's + # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a + # factor - however, each layer.conf is free to either prepend or append to + # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might + # not be exactly the order present in bblayers.conf either. + bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True)) + overlayed_class_found = False + for (classfile, classdirs) in classes.items(): + if len(classdirs) > 1: + if not overlayed_class_found: + logger.plain('=== Overlayed classes ===') + overlayed_class_found = True + + mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile)) + if args.filenames: + logger.plain('%s' % mainfile) + else: + # We effectively have to guess the layer here + logger.plain('%s:' % classfile) + mainlayername = '?' + for layerdir in self.bblayers: + classdir = os.path.join(layerdir, 'classes') + if mainfile.startswith(classdir): + mainlayername = self.get_layer_name(layerdir) + logger.plain(' %s' % mainlayername) + for classdir in classdirs: + fullpath = os.path.join(classdir, classfile) + if fullpath != mainfile: + if args.filenames: + print(' %s' % fullpath) + else: + print(' %s' % self.get_layer_name(os.path.dirname(classdir))) + + if overlayed_class_found: + items_listed = True; + + if not items_listed: + logger.plain('No overlayed files found.') + + + def do_show_recipes(self, args): + """list available recipes, showing the layer they are provided by + +Lists the names of recipes and the available versions in each +layer, with the preferred version first. Optionally you may specify +pnspec to match a specified recipe name (supports wildcards). Note that +skipped recipes will also be listed, with a " (skipped)" suffix. +""" + self.init_bbhandler() + + inheritlist = args.inherits.split(',') if args.inherits else [] + if inheritlist or args.pnspec or args.multiple: + title = 'Matching recipes:' + else: + title = 'Available recipes:' + self.list_recipes(title, args.pnspec, False, False, args.filenames, args.multiple, inheritlist) + + + def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only, inherits): + if inherits: + bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True)) + for classname in inherits: + classfile = 'classes/%s.bbclass' % classname + if not bb.utils.which(bbpath, classfile, history=False): + raise UserError('No class named %s found in BBPATH' % classfile) + + pkg_pn = self.bbhandler.cooker.recipecache.pkg_pn + (latest_versions, preferred_versions) = bb.providers.findProviders(self.bbhandler.config_data, self.bbhandler.cooker.recipecache, pkg_pn) + allproviders = bb.providers.allProviders(self.bbhandler.cooker.recipecache) + + # Ensure we list skipped recipes + # We are largely guessing about PN, PV and the preferred version here, + # but we have no choice since skipped recipes are not fully parsed + skiplist = self.bbhandler.cooker.skiplist.keys() + skiplist.sort( key=lambda fileitem: self.bbhandler.cooker.collection.calc_bbfile_priority(fileitem) ) + skiplist.reverse() + for fn in skiplist: + recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_') + p = recipe_parts[0] + if len(recipe_parts) > 1: + ver = (None, recipe_parts[1], None) + else: + ver = (None, 'unknown', None) + allproviders[p].append((ver, fn)) + if not p in pkg_pn: + pkg_pn[p] = 'dummy' + preferred_versions[p] = (ver, fn) + + def print_item(f, pn, ver, layer, ispref): + if f in skiplist: + skipped = ' (skipped)' + else: + skipped = '' + if show_filenames: + if ispref: + logger.plain("%s%s", f, skipped) + else: + logger.plain(" %s%s", f, skipped) + else: + if ispref: + logger.plain("%s:", pn) + logger.plain(" %s %s%s", layer.ljust(20), ver, skipped) + + global_inherit = (self.bbhandler.config_data.getVar('INHERIT', True) or "").split() + cls_re = re.compile('classes/') + + preffiles = [] + items_listed = False + for p in sorted(pkg_pn): + if pnspec: + if not fnmatch.fnmatch(p, pnspec): + continue + + if len(allproviders[p]) > 1 or not show_multi_provider_only: + pref = preferred_versions[p] + realfn = bb.cache.Cache.virtualfn2realfn(pref[1]) + preffile = realfn[0] + + # We only display once per recipe, we should prefer non extended versions of the + # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl + # which would otherwise sort first). + if realfn[1] and realfn[0] in self.bbhandler.cooker.recipecache.pkg_fn: + continue + + if inherits: + matchcount = 0 + recipe_inherits = self.bbhandler.cooker_data.inherits.get(preffile, []) + for cls in recipe_inherits: + if cls_re.match(cls): + continue + classname = os.path.splitext(os.path.basename(cls))[0] + if classname in global_inherit: + continue + elif classname in inherits: + matchcount += 1 + if matchcount != len(inherits): + # No match - skip this recipe + continue + + if preffile not in preffiles: + preflayer = self.get_file_layer(preffile) + multilayer = False + same_ver = True + provs = [] + for prov in allproviders[p]: + provfile = bb.cache.Cache.virtualfn2realfn(prov[1])[0] + provlayer = self.get_file_layer(provfile) + provs.append((provfile, provlayer, prov[0])) + if provlayer != preflayer: + multilayer = True + if prov[0] != pref[0]: + same_ver = False + + if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only): + if not items_listed: + logger.plain('=== %s ===' % title) + items_listed = True + print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True) + for (provfile, provlayer, provver) in provs: + if provfile != preffile: + print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False) + # Ensure we don't show two entries for BBCLASSEXTENDed recipes + preffiles.append(preffile) + + return items_listed + + + def do_flatten(self, args): + """flatten layer configuration into a separate output directory. + +Takes the specified layers (or all layers in the current layer +configuration if none are specified) and builds a "flattened" directory +containing the contents of all layers, with any overlayed recipes removed +and bbappends appended to the corresponding recipes. Note that some manual +cleanup may still be necessary afterwards, in particular: + +* where non-recipe files (such as patches) are overwritten (the flatten + command will show a warning for these) +* where anything beyond the normal layer setup has been added to + layer.conf (only the lowest priority number layer's layer.conf is used) +* overridden/appended items from bbappends will need to be tidied up +* when the flattened layers do not have the same directory structure (the + flatten command should show a warning when this will cause a problem) + +Warning: if you flatten several layers where another layer is intended to +be used "inbetween" them (in layer priority order) such that recipes / +bbappends in the layers interact, and then attempt to use the new output +layer together with that other layer, you may no longer get the same +build results (as the layer priority order has effectively changed). +""" + if len(args.layer) == 1: + logger.error('If you specify layers to flatten you must specify at least two') + return 1 + + outputdir = args.outputdir + if os.path.exists(outputdir) and os.listdir(outputdir): + logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir) + return 1 + + self.init_bbhandler() + layers = self.bblayers + if len(args.layer) > 2: + layernames = args.layer + found_layernames = [] + found_layerdirs = [] + for layerdir in layers: + layername = self.get_layer_name(layerdir) + if layername in layernames: + found_layerdirs.append(layerdir) + found_layernames.append(layername) + + for layername in layernames: + if not layername in found_layernames: + logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0]))) + return + layers = found_layerdirs + else: + layernames = [] + + # Ensure a specified path matches our list of layers + def layer_path_match(path): + for layerdir in layers: + if path.startswith(os.path.join(layerdir, '')): + return layerdir + return None + + applied_appends = [] + for layer in layers: + overlayed = [] + for f in self.bbhandler.cooker.collection.overlayed.iterkeys(): + for of in self.bbhandler.cooker.collection.overlayed[f]: + if of.startswith(layer): + overlayed.append(of) + + logger.plain('Copying files from %s...' % layer ) + for root, dirs, files in os.walk(layer): + for f1 in files: + f1full = os.sep.join([root, f1]) + if f1full in overlayed: + logger.plain(' Skipping overlayed file %s' % f1full ) + else: + ext = os.path.splitext(f1)[1] + if ext != '.bbappend': + fdest = f1full[len(layer):] + fdest = os.path.normpath(os.sep.join([outputdir,fdest])) + bb.utils.mkdirhier(os.path.dirname(fdest)) + if os.path.exists(fdest): + if f1 == 'layer.conf' and root.endswith('/conf'): + logger.plain(' Skipping layer config file %s' % f1full ) + continue + else: + logger.warn('Overwriting file %s', fdest) + bb.utils.copyfile(f1full, fdest) + if ext == '.bb': + for append in self.bbhandler.cooker.collection.get_file_appends(f1full): + if layer_path_match(append): + logger.plain(' Applying append %s to %s' % (append, fdest)) + self.apply_append(append, fdest) + applied_appends.append(append) + + # Take care of when some layers are excluded and yet we have included bbappends for those recipes + for b in self.bbhandler.cooker.collection.bbappends: + (recipename, appendname) = b + if appendname not in applied_appends: + first_append = None + layer = layer_path_match(appendname) + if layer: + if first_append: + self.apply_append(appendname, first_append) + else: + fdest = appendname[len(layer):] + fdest = os.path.normpath(os.sep.join([outputdir,fdest])) + bb.utils.mkdirhier(os.path.dirname(fdest)) + bb.utils.copyfile(appendname, fdest) + first_append = fdest + + # Get the regex for the first layer in our list (which is where the conf/layer.conf file will + # have come from) + first_regex = None + layerdir = layers[0] + for layername, pattern, regex, _ in self.bbhandler.cooker.recipecache.bbfile_config_priorities: + if regex.match(os.path.join(layerdir, 'test')): + first_regex = regex + break + + if first_regex: + # Find the BBFILES entries that match (which will have come from this conf/layer.conf file) + bbfiles = str(self.bbhandler.config_data.getVar('BBFILES', True)).split() + bbfiles_layer = [] + for item in bbfiles: + if first_regex.match(item): + newpath = os.path.join(outputdir, item[len(layerdir)+1:]) + bbfiles_layer.append(newpath) + + if bbfiles_layer: + # Check that all important layer files match BBFILES + for root, dirs, files in os.walk(outputdir): + for f1 in files: + ext = os.path.splitext(f1)[1] + if ext in ['.bb', '.bbappend']: + f1full = os.sep.join([root, f1]) + entry_found = False + for item in bbfiles_layer: + if fnmatch.fnmatch(f1full, item): + entry_found = True + break + if not entry_found: + logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full) + + def get_file_layer(self, filename): + layerdir = self.get_file_layerdir(filename) + if layerdir: + return self.get_layer_name(layerdir) + else: + return '?' + + def get_file_layerdir(self, filename): + layer = bb.utils.get_file_layer(filename, self.bbhandler.config_data) + return self.bbfile_collections.get(layer, None) + + def remove_layer_prefix(self, f): + """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the + return value will be: layer_dir/foo/blah""" + f_layerdir = self.get_file_layerdir(f) + if not f_layerdir: + return f + prefix = os.path.join(os.path.dirname(f_layerdir), '') + return f[len(prefix):] if f.startswith(prefix) else f + + def get_layer_name(self, layerdir): + return os.path.basename(layerdir.rstrip(os.sep)) + + def apply_append(self, appendname, recipename): + with open(appendname, 'r') as appendfile: + with open(recipename, 'a') as recipefile: + recipefile.write('\n') + recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname)) + recipefile.writelines(appendfile.readlines()) + + def do_show_appends(self, args): + """list bbappend files and recipe files they apply to + +Lists recipes with the bbappends that apply to them as subitems. +""" + self.init_bbhandler() + + logger.plain('=== Appended recipes ===') + + pnlist = list(self.bbhandler.cooker_data.pkg_pn.keys()) + pnlist.sort() + appends = False + for pn in pnlist: + if self.show_appends_for_pn(pn): + appends = True + + if self.show_appends_for_skipped(): + appends = True + + if not appends: + logger.plain('No append files found') + + def show_appends_for_pn(self, pn): + filenames = self.bbhandler.cooker_data.pkg_pn[pn] + + best = bb.providers.findBestProvider(pn, + self.bbhandler.config_data, + self.bbhandler.cooker_data, + self.bbhandler.cooker_data.pkg_pn) + best_filename = os.path.basename(best[3]) + + return self.show_appends_output(filenames, best_filename) + + def show_appends_for_skipped(self): + filenames = [os.path.basename(f) + for f in self.bbhandler.cooker.skiplist.iterkeys()] + return self.show_appends_output(filenames, None, " (skipped)") + + def show_appends_output(self, filenames, best_filename, name_suffix = ''): + appended, missing = self.get_appends_for_files(filenames) + if appended: + for basename, appends in appended: + logger.plain('%s%s:', basename, name_suffix) + for append in appends: + logger.plain(' %s', append) + + if best_filename: + if best_filename in missing: + logger.warn('%s: missing append for preferred version', + best_filename) + return True + else: + return False + + def get_appends_for_files(self, filenames): + appended, notappended = [], [] + for filename in filenames: + _, cls = bb.cache.Cache.virtualfn2realfn(filename) + if cls: + continue + + basename = os.path.basename(filename) + appends = self.bbhandler.cooker.collection.get_file_appends(basename) + if appends: + appended.append((basename, list(appends))) + else: + notappended.append(basename) + return appended, notappended + + def do_show_cross_depends(self, args): + """Show dependencies between recipes that cross layer boundaries. + +Figure out the dependencies between recipes that cross layer boundaries. + +NOTE: .bbappend files can impact the dependencies. +""" + ignore_layers = (args.ignore or '').split(',') + + self.init_bbhandler() + + pkg_fn = self.bbhandler.cooker_data.pkg_fn + bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True)) + self.require_re = re.compile(r"require\s+(.+)") + self.include_re = re.compile(r"include\s+(.+)") + self.inherit_re = re.compile(r"inherit\s+(.+)") + + global_inherit = (self.bbhandler.config_data.getVar('INHERIT', True) or "").split() + + # The bb's DEPENDS and RDEPENDS + for f in pkg_fn: + f = bb.cache.Cache.virtualfn2realfn(f)[0] + # Get the layername that the file is in + layername = self.get_file_layer(f) + + # The DEPENDS + deps = self.bbhandler.cooker_data.deps[f] + for pn in deps: + if pn in self.bbhandler.cooker_data.pkg_pn: + best = bb.providers.findBestProvider(pn, + self.bbhandler.config_data, + self.bbhandler.cooker_data, + self.bbhandler.cooker_data.pkg_pn) + self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers) + + # The RDPENDS + all_rdeps = self.bbhandler.cooker_data.rundeps[f].values() + # Remove the duplicated or null one. + sorted_rdeps = {} + # The all_rdeps is the list in list, so we need two for loops + for k1 in all_rdeps: + for k2 in k1: + sorted_rdeps[k2] = 1 + all_rdeps = sorted_rdeps.keys() + for rdep in all_rdeps: + all_p = bb.providers.getRuntimeProviders(self.bbhandler.cooker_data, rdep) + if all_p: + if f in all_p: + # The recipe provides this one itself, ignore + continue + best = bb.providers.filterProvidersRunTime(all_p, rdep, + self.bbhandler.config_data, + self.bbhandler.cooker_data)[0][0] + self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers) + + # The RRECOMMENDS + all_rrecs = self.bbhandler.cooker_data.runrecs[f].values() + # Remove the duplicated or null one. + sorted_rrecs = {} + # The all_rrecs is the list in list, so we need two for loops + for k1 in all_rrecs: + for k2 in k1: + sorted_rrecs[k2] = 1 + all_rrecs = sorted_rrecs.keys() + for rrec in all_rrecs: + all_p = bb.providers.getRuntimeProviders(self.bbhandler.cooker_data, rrec) + if all_p: + if f in all_p: + # The recipe provides this one itself, ignore + continue + best = bb.providers.filterProvidersRunTime(all_p, rrec, + self.bbhandler.config_data, + self.bbhandler.cooker_data)[0][0] + self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers) + + # The inherit class + cls_re = re.compile('classes/') + if f in self.bbhandler.cooker_data.inherits: + inherits = self.bbhandler.cooker_data.inherits[f] + for cls in inherits: + # The inherits' format is [classes/cls, /path/to/classes/cls] + # ignore the classes/cls. + if not cls_re.match(cls): + classname = os.path.splitext(os.path.basename(cls))[0] + if classname in global_inherit: + continue + inherit_layername = self.get_file_layer(cls) + if inherit_layername != layername and not inherit_layername in ignore_layers: + if not args.filenames: + f_short = self.remove_layer_prefix(f) + cls = self.remove_layer_prefix(cls) + else: + f_short = f + logger.plain("%s inherits %s" % (f_short, cls)) + + # The 'require/include xxx' in the bb file + pv_re = re.compile(r"\${PV}") + with open(f, 'r') as fnfile: + line = fnfile.readline() + while line: + m, keyword = self.match_require_include(line) + # Found the 'require/include xxxx' + if m: + needed_file = m.group(1) + # Replace the ${PV} with the real PV + if pv_re.search(needed_file) and f in self.bbhandler.cooker_data.pkg_pepvpr: + pv = self.bbhandler.cooker_data.pkg_pepvpr[f][1] + needed_file = re.sub(r"\${PV}", pv, needed_file) + self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers) + line = fnfile.readline() + + # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass + conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$") + inc_re = re.compile(".*\.inc$") + # The "inherit xxx" in .bbclass + bbclass_re = re.compile(".*\.bbclass$") + for layerdir in self.bblayers: + layername = self.get_layer_name(layerdir) + for dirpath, dirnames, filenames in os.walk(layerdir): + for name in filenames: + f = os.path.join(dirpath, name) + s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f) + if s: + with open(f, 'r') as ffile: + line = ffile.readline() + while line: + m, keyword = self.match_require_include(line) + # Only bbclass has the "inherit xxx" here. + bbclass="" + if not m and f.endswith(".bbclass"): + m, keyword = self.match_inherit(line) + bbclass=".bbclass" + # Find a 'require/include xxxx' + if m: + self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers) + line = ffile.readline() + + def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers): + """Print the depends that crosses a layer boundary""" + needed_file = bb.utils.which(bbpath, needed_filename) + if needed_file: + # Which layer is this file from + needed_layername = self.get_file_layer(needed_file) + if needed_layername != layername and not needed_layername in ignore_layers: + if not show_filenames: + f = self.remove_layer_prefix(f) + needed_file = self.remove_layer_prefix(needed_file) + logger.plain("%s %s %s" %(f, keyword, needed_file)) + + def match_inherit(self, line): + """Match the inherit xxx line""" + return (self.inherit_re.match(line), "inherits") + + def match_require_include(self, line): + """Match the require/include xxx line""" + m = self.require_re.match(line) + keyword = "requires" + if not m: + m = self.include_re.match(line) + keyword = "includes" + return (m, keyword) + + def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers): + """Print the DEPENDS/RDEPENDS file that crosses a layer boundary""" + best_realfn = bb.cache.Cache.virtualfn2realfn(needed_file)[0] + needed_layername = self.get_file_layer(best_realfn) + if needed_layername != layername and not needed_layername in ignore_layers: + if not show_filenames: + f = self.remove_layer_prefix(f) + best_realfn = self.remove_layer_prefix(best_realfn) + + logger.plain("%s %s %s" % (f, keyword, best_realfn)) + + +def main(): + + cmds = Commands() + + def add_command(cmdname, function, *args, **kwargs): + # Convert docstring for function to help (one-liner shown in main --help) and description (shown in subcommand --help) + docsplit = function.__doc__.splitlines() + help = docsplit[0] + if len(docsplit) > 1: + desc = '\n'.join(docsplit[1:]) + else: + desc = help + subparser = subparsers.add_parser(cmdname, *args, help=help, description=desc, formatter_class=argparse.RawTextHelpFormatter, **kwargs) + subparser.set_defaults(func=function) + return subparser + + parser = argparse.ArgumentParser(description="BitBake layers utility", + epilog="Use %(prog)s --help to get help on a specific command") + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') + subparsers = parser.add_subparsers(title='subcommands', metavar='') + + parser_show_layers = add_command('show-layers', cmds.do_show_layers) + + parser_add_layer = add_command('add-layer', cmds.do_add_layer) + parser_add_layer.add_argument('layerdir', help='Layer directory to add') + + parser_remove_layer = add_command('remove-layer', cmds.do_remove_layer) + parser_remove_layer.add_argument('layerdir', help='Layer directory to remove (wildcards allowed, enclose in quotes to avoid shell expansion)') + parser_remove_layer.set_defaults(func=cmds.do_remove_layer) + + parser_show_overlayed = add_command('show-overlayed', cmds.do_show_overlayed) + parser_show_overlayed.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') + parser_show_overlayed.add_argument('-s', '--same-version', help='only list overlayed recipes where the version is the same', action='store_true') + + parser_show_recipes = add_command('show-recipes', cmds.do_show_recipes) + parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') + parser_show_recipes.add_argument('-m', '--multiple', help='only list where multiple recipes (in the same layer or different layers) exist for the same recipe name', action='store_true') + parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class', metavar='CLASS', default='') + parser_show_recipes.add_argument('pnspec', nargs='?', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') + + parser_show_appends = add_command('show-appends', cmds.do_show_appends) + + parser_flatten = add_command('flatten', cmds.do_flatten) + parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)') + parser_flatten.add_argument('outputdir', help='Output directory') + + parser_show_cross_depends = add_command('show-cross-depends', cmds.do_show_cross_depends) + parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true') + parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME') + + parser_layerindex_fetch = add_command('layerindex-fetch', cmds.do_layerindex_fetch) + parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true') + parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master') + parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER') + parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch') + + parser_layerindex_show_depends = add_command('layerindex-show-depends', cmds.do_layerindex_show_depends) + parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master') + parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query') + + args = parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + elif args.quiet: + logger.setLevel(logging.ERROR) + + try: + ret = args.func(args) + except UserError as err: + logger.error(str(err)) + ret = 1 + + return ret + + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake-prserv b/import-layers/yocto-poky/bitbake/bin/bitbake-prserv new file mode 100755 index 000000000..03821446b --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/bitbake-prserv @@ -0,0 +1,55 @@ +#!/usr/bin/env python +import os +import sys,logging +import optparse + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib')) + +import prserv +import prserv.serv + +__version__="1.0.0" + +PRHOST_DEFAULT='0.0.0.0' +PRPORT_DEFAULT=8585 + +def main(): + parser = optparse.OptionParser( + version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__), + usage = "%prog < --start | --stop > [options]") + + parser.add_option("-f", "--file", help="database filename(default: prserv.sqlite3)", action="store", + dest="dbfile", type="string", default="prserv.sqlite3") + parser.add_option("-l", "--log", help="log filename(default: prserv.log)", action="store", + dest="logfile", type="string", default="prserv.log") + parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG", + action = "store", type="string", dest="loglevel", default = "INFO") + parser.add_option("--start", help="start daemon", + action="store_true", dest="start") + parser.add_option("--stop", help="stop daemon", + action="store_true", dest="stop") + parser.add_option("--host", help="ip address to bind", action="store", + dest="host", type="string", default=PRHOST_DEFAULT) + parser.add_option("--port", help="port number(default: 8585)", action="store", + dest="port", type="int", default=PRPORT_DEFAULT) + + options, args = parser.parse_args(sys.argv) + prserv.init_logger(os.path.abspath(options.logfile),options.loglevel) + + if options.start: + ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile)) + elif options.stop: + ret=prserv.serv.stop_daemon(options.host, options.port) + else: + ret=parser.print_help() + return ret + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) + diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake-selftest b/import-layers/yocto-poky/bitbake/bin/bitbake-selftest new file mode 100755 index 000000000..462eb1b2b --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/bitbake-selftest @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# +# Copyright (C) 2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys, logging +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) + +import unittest +try: + import bb +except RuntimeError as exc: + sys.exit(str(exc)) + +def usage(): + print('usage: [BB_SKIP_NETTESTS=yes] %s [-v] [testname1 [testname2]...]' % os.path.basename(sys.argv[0])) + +verbosity = 1 + +tests = sys.argv[1:] +if '-v' in sys.argv: + tests.remove('-v') + verbosity = 2 + +if tests: + if '--help' in sys.argv[1:]: + usage() + sys.exit(0) +else: + tests = ["bb.tests.codeparser", + "bb.tests.cow", + "bb.tests.data", + "bb.tests.fetch", + "bb.tests.parse", + "bb.tests.utils"] + +for t in tests: + t = '.'.join(t.split('.')[:3]) + __import__(t) + +unittest.main(argv=["bitbake-selftest"] + tests, verbosity=verbosity) + diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake-worker b/import-layers/yocto-poky/bitbake/bin/bitbake-worker new file mode 100755 index 000000000..767a1c033 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/bitbake-worker @@ -0,0 +1,447 @@ +#!/usr/bin/env python + +import os +import sys +import warnings +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) +from bb import fetch2 +import logging +import bb +import select +import errno +import signal +from multiprocessing import Lock + +# Users shouldn't be running this code directly +if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"): + print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.") + sys.exit(1) + +profiling = False +if sys.argv[1].startswith("decafbadbad"): + profiling = True + try: + import cProfile as profile + except: + import profile + +# Unbuffer stdout to avoid log truncation in the event +# of an unorderly exit as well as to provide timely +# updates to log files for use with tail +try: + if sys.stdout.name == '': + sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) +except: + pass + +logger = logging.getLogger("BitBake") + +try: + import cPickle as pickle +except ImportError: + import pickle + bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.") + + +worker_pipe = sys.stdout.fileno() +bb.utils.nonblockingfd(worker_pipe) +# Need to guard against multiprocessing being used in child processes +# and multiple processes trying to write to the parent at the same time +worker_pipe_lock = None + +handler = bb.event.LogHandler() +logger.addHandler(handler) + +if 0: + # Code to write out a log file of all events passing through the worker + logfilename = "/tmp/workerlogfile" + format_str = "%(levelname)s: %(message)s" + conlogformat = bb.msg.BBLogFormatter(format_str) + consolelog = logging.FileHandler(logfilename) + bb.msg.addDefaultlogFilter(consolelog) + consolelog.setFormatter(conlogformat) + logger.addHandler(consolelog) + +worker_queue = "" + +def worker_fire(event, d): + data = "" + pickle.dumps(event) + "" + worker_fire_prepickled(data) + +def worker_fire_prepickled(event): + global worker_queue + + worker_queue = worker_queue + event + worker_flush() + +def worker_flush(): + global worker_queue, worker_pipe + + if not worker_queue: + return + + try: + written = os.write(worker_pipe, worker_queue) + worker_queue = worker_queue[written:] + except (IOError, OSError) as e: + if e.errno != errno.EAGAIN and e.errno != errno.EPIPE: + raise + +def worker_child_fire(event, d): + global worker_pipe + global worker_pipe_lock + + data = "" + pickle.dumps(event) + "" + try: + worker_pipe_lock.acquire() + worker_pipe.write(data) + worker_pipe_lock.release() + except IOError: + sigterm_handler(None, None) + raise + +bb.event.worker_fire = worker_fire + +lf = None +#lf = open("/tmp/workercommandlog", "w+") +def workerlog_write(msg): + if lf: + lf.write(msg) + lf.flush() + +def sigterm_handler(signum, frame): + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.killpg(0, signal.SIGTERM) + sys.exit() + +def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False): + # We need to setup the environment BEFORE the fork, since + # a fork() or exec*() activates PSEUDO... + + envbackup = {} + fakeenv = {} + umask = None + + taskdep = workerdata["taskdeps"][fn] + if 'umask' in taskdep and taskname in taskdep['umask']: + # umask might come in as a number or text string.. + try: + umask = int(taskdep['umask'][taskname],8) + except TypeError: + umask = taskdep['umask'][taskname] + + # We can't use the fakeroot environment in a dry run as it possibly hasn't been built + if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not cfg.dry_run: + envvars = (workerdata["fakerootenv"][fn] or "").split() + for key, value in (var.split('=') for var in envvars): + envbackup[key] = os.environ.get(key) + os.environ[key] = value + fakeenv[key] = value + + fakedirs = (workerdata["fakerootdirs"][fn] or "").split() + for p in fakedirs: + bb.utils.mkdirhier(p) + logger.debug(2, 'Running %s:%s under fakeroot, fakedirs: %s' % + (fn, taskname, ', '.join(fakedirs))) + else: + envvars = (workerdata["fakerootnoenv"][fn] or "").split() + for key, value in (var.split('=') for var in envvars): + envbackup[key] = os.environ.get(key) + os.environ[key] = value + fakeenv[key] = value + + sys.stdout.flush() + sys.stderr.flush() + + try: + pipein, pipeout = os.pipe() + pipein = os.fdopen(pipein, 'rb', 4096) + pipeout = os.fdopen(pipeout, 'wb', 0) + pid = os.fork() + except OSError as e: + bb.msg.fatal("RunQueue", "fork failed: %d (%s)" % (e.errno, e.strerror)) + + if pid == 0: + def child(): + global worker_pipe + global worker_pipe_lock + pipein.close() + + signal.signal(signal.SIGTERM, sigterm_handler) + # Let SIGHUP exit as SIGTERM + signal.signal(signal.SIGHUP, sigterm_handler) + bb.utils.signal_on_parent_exit("SIGTERM") + + # Save out the PID so that the event can include it the + # events + bb.event.worker_pid = os.getpid() + bb.event.worker_fire = worker_child_fire + worker_pipe = pipeout + worker_pipe_lock = Lock() + + # Make the child the process group leader and ensure no + # child process will be controlled by the current terminal + # This ensures signals sent to the controlling terminal like Ctrl+C + # don't stop the child processes. + os.setsid() + # No stdin + newsi = os.open(os.devnull, os.O_RDWR) + os.dup2(newsi, sys.stdin.fileno()) + + if umask: + os.umask(umask) + + data.setVar("BB_WORKERCONTEXT", "1") + data.setVar("BB_TASKDEPDATA", taskdepdata) + data.setVar("BUILDNAME", workerdata["buildname"]) + data.setVar("DATE", workerdata["date"]) + data.setVar("TIME", workerdata["time"]) + bb.parse.siggen.set_taskdata(workerdata["sigdata"]) + ret = 0 + try: + the_data = bb.cache.Cache.loadDataFull(fn, appends, data) + the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task]) + + bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN", True), taskname.replace("do_", ""))) + + # exported_vars() returns a generator which *cannot* be passed to os.environ.update() + # successfully. We also need to unset anything from the environment which shouldn't be there + exports = bb.data.exported_vars(the_data) + bb.utils.empty_environment() + for e, v in exports: + os.environ[e] = v + for e in fakeenv: + os.environ[e] = fakeenv[e] + the_data.setVar(e, fakeenv[e]) + the_data.setVarFlag(e, 'export', "1") + + if quieterrors: + the_data.setVarFlag(taskname, "quieterrors", "1") + + except Exception as exc: + if not quieterrors: + logger.critical(str(exc)) + os._exit(1) + try: + if cfg.dry_run: + return 0 + return bb.build.exec_task(fn, taskname, the_data, cfg.profile) + except: + os._exit(1) + if not profiling: + os._exit(child()) + else: + profname = "profile-%s.log" % (fn.replace("/", "-") + "-" + taskname) + prof = profile.Profile() + try: + ret = profile.Profile.runcall(prof, child) + finally: + prof.dump_stats(profname) + bb.utils.process_profilelog(profname) + os._exit(ret) + else: + for key, value in envbackup.iteritems(): + if value is None: + del os.environ[key] + else: + os.environ[key] = value + + return pid, pipein, pipeout + +class runQueueWorkerPipe(): + """ + Abstraction for a pipe between a worker thread and the worker server + """ + def __init__(self, pipein, pipeout): + self.input = pipein + if pipeout: + pipeout.close() + bb.utils.nonblockingfd(self.input) + self.queue = "" + + def read(self): + start = len(self.queue) + try: + self.queue = self.queue + self.input.read(102400) + except (OSError, IOError) as e: + if e.errno != errno.EAGAIN: + raise + + end = len(self.queue) + index = self.queue.find("") + while index != -1: + worker_fire_prepickled(self.queue[:index+8]) + self.queue = self.queue[index+8:] + index = self.queue.find("") + return (end > start) + + def close(self): + while self.read(): + continue + if len(self.queue) > 0: + print("Warning, worker child left partial message: %s" % self.queue) + self.input.close() + +normalexit = False + +class BitbakeWorker(object): + def __init__(self, din): + self.input = din + bb.utils.nonblockingfd(self.input) + self.queue = "" + self.cookercfg = None + self.databuilder = None + self.data = None + self.build_pids = {} + self.build_pipes = {} + + signal.signal(signal.SIGTERM, self.sigterm_exception) + # Let SIGHUP exit as SIGTERM + signal.signal(signal.SIGHUP, self.sigterm_exception) + if "beef" in sys.argv[1]: + bb.utils.set_process_name("Worker (Fakeroot)") + else: + bb.utils.set_process_name("Worker") + + def sigterm_exception(self, signum, stackframe): + if signum == signal.SIGTERM: + bb.warn("Worker received SIGTERM, shutting down...") + elif signum == signal.SIGHUP: + bb.warn("Worker received SIGHUP, shutting down...") + self.handle_finishnow(None) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGTERM) + + def serve(self): + while True: + (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1) + if self.input in ready: + try: + r = self.input.read() + if len(r) == 0: + # EOF on pipe, server must have terminated + self.sigterm_exception(signal.SIGTERM, None) + self.queue = self.queue + r + except (OSError, IOError): + pass + if len(self.queue): + self.handle_item("cookerconfig", self.handle_cookercfg) + self.handle_item("workerdata", self.handle_workerdata) + self.handle_item("runtask", self.handle_runtask) + self.handle_item("finishnow", self.handle_finishnow) + self.handle_item("ping", self.handle_ping) + self.handle_item("quit", self.handle_quit) + + for pipe in self.build_pipes: + self.build_pipes[pipe].read() + if len(self.build_pids): + self.process_waitpid() + worker_flush() + + + def handle_item(self, item, func): + if self.queue.startswith("<" + item + ">"): + index = self.queue.find("") + while index != -1: + func(self.queue[(len(item) + 2):index]) + self.queue = self.queue[(index + len(item) + 3):] + index = self.queue.find("") + + def handle_cookercfg(self, data): + self.cookercfg = pickle.loads(data) + self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True) + self.databuilder.parseBaseConfiguration() + self.data = self.databuilder.data + + def handle_workerdata(self, data): + self.workerdata = pickle.loads(data) + bb.msg.loggerDefaultDebugLevel = self.workerdata["logdefaultdebug"] + bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"] + bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"] + bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"] + self.data.setVar("PRSERV_HOST", self.workerdata["prhost"]) + + def handle_ping(self, _): + workerlog_write("Handling ping\n") + + logger.warn("Pong from bitbake-worker!") + + def handle_quit(self, data): + workerlog_write("Handling quit\n") + + global normalexit + normalexit = True + sys.exit(0) + + def handle_runtask(self, data): + fn, task, taskname, quieterrors, appends, taskdepdata = pickle.loads(data) + workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname)) + + pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.workerdata, fn, task, taskname, appends, taskdepdata, quieterrors) + + self.build_pids[pid] = task + self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout) + + def process_waitpid(self): + """ + Return none is there are no processes awaiting result collection, otherwise + collect the process exit codes and close the information pipe. + """ + try: + pid, status = os.waitpid(-1, os.WNOHANG) + if pid == 0 or os.WIFSTOPPED(status): + return None + except OSError: + return None + + workerlog_write("Exit code of %s for pid %s\n" % (status, pid)) + + if os.WIFEXITED(status): + status = os.WEXITSTATUS(status) + elif os.WIFSIGNALED(status): + # Per shell conventions for $?, when a process exits due to + # a signal, we return an exit code of 128 + SIGNUM + status = 128 + os.WTERMSIG(status) + + task = self.build_pids[pid] + del self.build_pids[pid] + + self.build_pipes[pid].close() + del self.build_pipes[pid] + + worker_fire_prepickled("" + pickle.dumps((task, status)) + "") + + def handle_finishnow(self, _): + if self.build_pids: + logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids)) + for k, v in self.build_pids.iteritems(): + try: + os.kill(-k, signal.SIGTERM) + os.waitpid(-1, 0) + except: + pass + for pipe in self.build_pipes: + self.build_pipes[pipe].read() + +try: + worker = BitbakeWorker(sys.stdin) + if not profiling: + worker.serve() + else: + profname = "profile-worker.log" + prof = profile.Profile() + try: + profile.Profile.runcall(prof, worker.serve) + finally: + prof.dump_stats(profname) + bb.utils.process_profilelog(profname) +except BaseException as e: + if not normalexit: + import traceback + sys.stderr.write(traceback.format_exc()) + sys.stderr.write(str(e)) +while len(worker_queue): + worker_flush() +workerlog_write("exitting") +sys.exit(0) + diff --git a/import-layers/yocto-poky/bitbake/bin/bitdoc b/import-layers/yocto-poky/bitbake/bin/bitdoc new file mode 100755 index 000000000..defb3dd37 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/bitdoc @@ -0,0 +1,531 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2005 Holger Hans Peter Freyther +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import optparse, os, sys + +# bitbake +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__), 'lib')) +import bb +import bb.parse +from string import split, join + +__version__ = "0.0.2" + +class HTMLFormatter: + """ + Simple class to help to generate some sort of HTML files. It is + quite inferior solution compared to docbook, gtkdoc, doxygen but it + should work for now. + We've a global introduction site (index.html) and then one site for + the list of keys (alphabetical sorted) and one for the list of groups, + one site for each key with links to the relations and groups. + + index.html + all_keys.html + all_groups.html + groupNAME.html + keyNAME.html + """ + + def replace(self, text, *pairs): + """ + From pydoc... almost identical at least + """ + while pairs: + (a, b) = pairs[0] + text = join(split(text, a), b) + pairs = pairs[1:] + return text + def escape(self, text): + """ + Escape string to be conform HTML + """ + return self.replace(text, + ('&', '&'), + ('<', '<' ), + ('>', '>' ) ) + def createNavigator(self): + """ + Create the navgiator + """ + return """ + + + + +
HomeGroupsKeys
+""" + + def relatedKeys(self, item): + """ + Create HTML to link to foreign keys + """ + + if len(item.related()) == 0: + return "" + + txt = "

See also:
" + txts = [] + for it in item.related(): + txts.append("""%(it)s""" % vars() ) + + return txt + ",".join(txts) + + def groups(self, item): + """ + Create HTML to link to related groups + """ + + if len(item.groups()) == 0: + return "" + + + txt = "

See also:
" + txts = [] + for group in item.groups(): + txts.append( """%s """ % (group, group) ) + + return txt + ",".join(txts) + + + def createKeySite(self, item): + """ + Create a site for a key. It contains the header/navigator, a heading, + the description, links to related keys and to the groups. + """ + + return """ +Key %s + + +%s +

%s

+ +
+

Synopsis

+

+%s +

+
+ +
+

Related Keys

+

+%s +

+
+ +
+

Groups

+

+%s +

+
+ + + +""" % (item.name(), self.createNavigator(), item.name(), + self.escape(item.description()), self.relatedKeys(item), self.groups(item)) + + def createGroupsSite(self, doc): + """ + Create the Group Overview site + """ + + groups = "" + sorted_groups = sorted(doc.groups()) + for group in sorted_groups: + groups += """%s
""" % (group, group) + + return """ +Group overview + + +%s +

Available Groups

+%s + +""" % (self.createNavigator(), groups) + + def createIndex(self): + """ + Create the index file + """ + + return """ +Bitbake Documentation + + +%s +

Documentation Entrance

+All available groups
+All available keys
+ +""" % self.createNavigator() + + def createKeysSite(self, doc): + """ + Create Overview of all avilable keys + """ + keys = "" + sorted_keys = sorted(doc.doc_keys()) + for key in sorted_keys: + keys += """%s
""" % (key, key) + + return """ +Key overview + + +%s +

Available Keys

+%s + +""" % (self.createNavigator(), keys) + + def createGroupSite(self, gr, items, _description = None): + """ + Create a site for a group: + Group the name of the group, items contain the name of the keys + inside this group + """ + groups = "" + description = "" + + # create a section with the group descriptions + if _description: + description += "

" % gr + description += _description + + items.sort(lambda x, y:cmp(x.name(), y.name())) + for group in items: + groups += """%s
""" % (group.name(), group.name()) + + return """ +Group %s + + +%s +%s +
+

Keys in Group %s

+
+%s
+
+
+ +""" % (gr, self.createNavigator(), description, gr, groups) + + + + def createCSS(self): + """ + Create the CSS file + """ + return """.synopsis, .classsynopsis +{ + background: #eeeeee; + border: solid 1px #aaaaaa; + padding: 0.5em; +} +.programlisting +{ + background: #eeeeff; + border: solid 1px #aaaaff; + padding: 0.5em; +} +.variablelist +{ + padding: 4px; + margin-left: 3em; +} +.variablelist td:first-child +{ + vertical-align: top; +} +table.navigation +{ + background: #ffeeee; + border: solid 1px #ffaaaa; + margin-top: 0.5em; + margin-bottom: 0.5em; +} +.navigation a +{ + color: #770000; +} +.navigation a:visited +{ + color: #550000; +} +.navigation .title +{ + font-size: 200%; +} +div.refnamediv +{ + margin-top: 2em; +} +div.gallery-float +{ + float: left; + padding: 10px; +} +div.gallery-float img +{ + border-style: none; +} +div.gallery-spacer +{ + clear: both; +} +a +{ + text-decoration: none; +} +a:hover +{ + text-decoration: underline; + color: #FF0000; +} +""" + + + +class DocumentationItem: + """ + A class to hold information about a configuration + item. It contains the key name, description, a list of related names, + and the group this item is contained in. + """ + + def __init__(self): + self._groups = [] + self._related = [] + self._name = "" + self._desc = "" + + def groups(self): + return self._groups + + def name(self): + return self._name + + def description(self): + return self._desc + + def related(self): + return self._related + + def setName(self, name): + self._name = name + + def setDescription(self, desc): + self._desc = desc + + def addGroup(self, group): + self._groups.append(group) + + def addRelation(self, relation): + self._related.append(relation) + + def sort(self): + self._related.sort() + self._groups.sort() + + +class Documentation: + """ + Holds the documentation... with mappings from key to items... + """ + + def __init__(self): + self.__keys = {} + self.__groups = {} + + def insert_doc_item(self, item): + """ + Insert the Doc Item into the internal list + of representation + """ + item.sort() + self.__keys[item.name()] = item + + for group in item.groups(): + if not group in self.__groups: + self.__groups[group] = [] + self.__groups[group].append(item) + self.__groups[group].sort() + + + def doc_item(self, key): + """ + Return the DocumentationInstance describing the key + """ + try: + return self.__keys[key] + except KeyError: + return None + + def doc_keys(self): + """ + Return the documented KEYS (names) + """ + return self.__keys.keys() + + def groups(self): + """ + Return the names of available groups + """ + return self.__groups.keys() + + def group_content(self, group_name): + """ + Return a list of keys/names that are in a specefic + group or the empty list + """ + try: + return self.__groups[group_name] + except KeyError: + return [] + + +def parse_cmdline(args): + """ + Parse the CMD line and return the result as a n-tuple + """ + + parser = optparse.OptionParser( version = "Bitbake Documentation Tool Core version %s, %%prog version %s" % (bb.__version__, __version__)) + usage = """%prog [options] + +Create a set of html pages (documentation) for a bitbake.conf.... +""" + + # Add the needed options + parser.add_option( "-c", "--config", help = "Use the specified configuration file as source", + action = "store", dest = "config", default = os.path.join("conf", "documentation.conf") ) + + parser.add_option( "-o", "--output", help = "Output directory for html files", + action = "store", dest = "output", default = "html/" ) + + parser.add_option( "-D", "--debug", help = "Increase the debug level", + action = "count", dest = "debug", default = 0 ) + + parser.add_option( "-v", "--verbose", help = "output more chit-char to the terminal", + action = "store_true", dest = "verbose", default = False ) + + options, args = parser.parse_args( sys.argv ) + + bb.msg.init_msgconfig(options.verbose, options.debug) + + return options.config, options.output + +def main(): + """ + The main Method + """ + + (config_file, output_dir) = parse_cmdline( sys.argv ) + + # right to let us load the file now + try: + documentation = bb.parse.handle( config_file, bb.data.init() ) + except IOError: + bb.fatal( "Unable to open %s" % config_file ) + except bb.parse.ParseError: + bb.fatal( "Unable to parse %s" % config_file ) + + if isinstance(documentation, dict): + documentation = documentation[""] + + # Assuming we've the file loaded now, we will initialize the 'tree' + doc = Documentation() + + # defined states + state_begin = 0 + state_see = 1 + state_group = 2 + + for key in bb.data.keys(documentation): + data = documentation.getVarFlag(key, "doc", False) + if not data: + continue + + # The Documentation now starts + doc_ins = DocumentationItem() + doc_ins.setName(key) + + + tokens = data.split(' ') + state = state_begin + string= "" + for token in tokens: + token = token.strip(',') + + if not state == state_see and token == "@see": + state = state_see + continue + elif not state == state_group and token == "@group": + state = state_group + continue + + if state == state_begin: + string += " %s" % token + elif state == state_see: + doc_ins.addRelation(token) + elif state == state_group: + doc_ins.addGroup(token) + + # set the description + doc_ins.setDescription(string) + doc.insert_doc_item(doc_ins) + + # let us create the HTML now + bb.utils.mkdirhier(output_dir) + os.chdir(output_dir) + + # Let us create the sites now. We do it in the following order + # Start with the index.html. It will point to sites explaining all + # keys and groups + html_slave = HTMLFormatter() + + f = file('style.css', 'w') + print >> f, html_slave.createCSS() + + f = file('index.html', 'w') + print >> f, html_slave.createIndex() + + f = file('all_groups.html', 'w') + print >> f, html_slave.createGroupsSite(doc) + + f = file('all_keys.html', 'w') + print >> f, html_slave.createKeysSite(doc) + + # now for each group create the site + for group in doc.groups(): + f = file('group%s.html' % group, 'w') + print >> f, html_slave.createGroupSite(group, doc.group_content(group)) + + # now for the keys + for key in doc.doc_keys(): + f = file('key%s.html' % doc.doc_item(key).name(), 'w') + print >> f, html_slave.createKeySite(doc.doc_item(key)) + + +if __name__ == "__main__": + main() diff --git a/import-layers/yocto-poky/bitbake/bin/image-writer b/import-layers/yocto-poky/bitbake/bin/image-writer new file mode 100755 index 000000000..e30ab45e3 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/image-writer @@ -0,0 +1,122 @@ +#!/usr/bin/env python + +# Copyright (c) 2012 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import sys +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname( \ + os.path.abspath(__file__))), 'lib')) +try: + import bb +except RuntimeError as exc: + sys.exit(str(exc)) + +import gtk +import optparse +import pygtk + +from bb.ui.crumbs.hobwidget import HobAltButton, HobButton +from bb.ui.crumbs.hig.crumbsmessagedialog import CrumbsMessageDialog +from bb.ui.crumbs.hig.deployimagedialog import DeployImageDialog +from bb.ui.crumbs.hig.imageselectiondialog import ImageSelectionDialog + +# I put all the fs bitbake supported here. Need more test. +DEPLOYABLE_IMAGE_TYPES = ["jffs2", "cramfs", "ext2", "ext3", "ext4", "btrfs", "squashfs", "ubi", "vmdk"] +Title = "USB Image Writer" + +class DeployWindow(gtk.Window): + def __init__(self, image_path=''): + super(DeployWindow, self).__init__() + + if len(image_path) > 0: + valid = True + if not os.path.exists(image_path): + valid = False + lbl = "Invalid image file path: %s.\nPress Select Image to select an image." % image_path + else: + image_path = os.path.abspath(image_path) + extend_name = os.path.splitext(image_path)[1][1:] + if extend_name not in DEPLOYABLE_IMAGE_TYPES: + valid = False + lbl = "Undeployable imge type: %s\nPress Select Image to select an image." % extend_name + + if not valid: + image_path = '' + crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO) + button = crumbs_dialog.add_button("Close", gtk.RESPONSE_OK) + HobButton.style_button(button) + crumbs_dialog.run() + crumbs_dialog.destroy() + + self.deploy_dialog = DeployImageDialog(Title, image_path, self, + gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT + | gtk.DIALOG_NO_SEPARATOR, None, standalone=True) + close_button = self.deploy_dialog.add_button("Close", gtk.RESPONSE_NO) + HobAltButton.style_button(close_button) + close_button.connect('clicked', gtk.main_quit) + + write_button = self.deploy_dialog.add_button("Write USB image", gtk.RESPONSE_YES) + HobAltButton.style_button(write_button) + + self.deploy_dialog.connect('select_image_clicked', self.select_image_clicked_cb) + self.deploy_dialog.connect('destroy', gtk.main_quit) + response = self.deploy_dialog.show() + + def select_image_clicked_cb(self, dialog): + cwd = os.getcwd() + dialog = ImageSelectionDialog(cwd, DEPLOYABLE_IMAGE_TYPES, Title, self, gtk.FILE_CHOOSER_ACTION_SAVE ) + button = dialog.add_button("Cancel", gtk.RESPONSE_NO) + HobAltButton.style_button(button) + button = dialog.add_button("Open", gtk.RESPONSE_YES) + HobAltButton.style_button(button) + response = dialog.run() + + if response == gtk.RESPONSE_YES: + if not dialog.image_names: + lbl = "No selections made\nClicked the radio button to select a image." + crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO) + button = crumbs_dialog.add_button("Close", gtk.RESPONSE_OK) + HobButton.style_button(button) + crumbs_dialog.run() + crumbs_dialog.destroy() + dialog.destroy() + return + + # get the full path of image + image_path = os.path.join(dialog.image_folder, dialog.image_names[0]) + self.deploy_dialog.set_image_text_buffer(image_path) + self.deploy_dialog.set_image_path(image_path) + + dialog.destroy() + +def main(): + parser = optparse.OptionParser( + usage = """%prog [-h] [image_file] + +%prog writes bootable images to USB devices. You can +provide the image file on the command line or select it using the GUI.""") + + options, args = parser.parse_args(sys.argv) + image_file = args[1] if len(args) > 1 else '' + dw = DeployWindow(image_file) + +if __name__ == '__main__': + try: + main() + gtk.main() + except Exception: + import traceback + traceback.print_exc() diff --git a/import-layers/yocto-poky/bitbake/bin/toaster b/import-layers/yocto-poky/bitbake/bin/toaster new file mode 100755 index 000000000..70c66d2c2 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/toaster @@ -0,0 +1,276 @@ +#!/bin/echo ERROR: This script needs to be sourced. Please run as . + +# toaster - shell script to start Toaster + +# Copyright (C) 2013-2015 Intel Corp. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +# Usage: source toaster [start|stop] +# [webport=] [noui] [noweb] + +# Helper function to kill a background toaster development server + +webserverKillAll() +{ + local pidfile + for pidfile in ${BUILDDIR}/.toastermain.pid ${BUILDDIR}/.runbuilds.pid; do + if [ -f ${pidfile} ]; then + pid=`cat ${pidfile}` + while kill -0 $pid 2>/dev/null; do + kill -SIGTERM -$pid 2>/dev/null + sleep 1 + # Kill processes if they are still running - may happen + # in interactive shells + ps fux | grep "python.*manage.py runserver" | awk '{print $2}' | xargs kill + done + rm ${pidfile} + fi + done +} + +webserverStartAll() +{ + # do not start if toastermain points to a valid process + if ! cat "${BUILDDIR}/.toastermain.pid" 2>/dev/null | xargs -I{} kill -0 {} ; then + retval=1 + rm "${BUILDDIR}/.toastermain.pid" + fi + + retval=0 + # you can always add a superuser later via + # ../bitbake/lib/toaster/manage.py createsuperuser --username= + $MANAGE migrate --noinput || retval=1 + + if [ $retval -eq 1 ]; then + echo "Failed migrations, aborting system start" 1>&2 + return $retval + fi + + $MANAGE checksettings --traceback || retval=1 + + if [ $retval -eq 1 ]; then + printf "\nError while checking settings; aborting\n" + return $retval + fi + + echo "Starting webserver..." + + $MANAGE runserver "0.0.0.0:$WEB_PORT" \ + >${BUILDDIR}/toaster_web.log 2>&1 \ + & echo $! >${BUILDDIR}/.toastermain.pid + + sleep 1 + + if ! cat "${BUILDDIR}/.toastermain.pid" | xargs -I{} kill -0 {} ; then + retval=1 + rm "${BUILDDIR}/.toastermain.pid" + else + echo "Webserver address: http://0.0.0.0:$WEB_PORT/" + fi + + return $retval +} + +INSTOPSYSTEM=0 + +# define the stop command +stop_system() +{ + # prevent reentry + if [ $INSTOPSYSTEM -eq 1 ]; then return; fi + INSTOPSYSTEM=1 + if [ -f ${BUILDDIR}/.toasterui.pid ]; then + kill `cat ${BUILDDIR}/.toasterui.pid` 2>/dev/null + rm ${BUILDDIR}/.toasterui.pid + fi + webserverKillAll + # unset exported variables + unset DATABASE_URL + unset TOASTER_CONF + unset TOASTER_DIR + unset BITBAKE_UI + unset BBBASEDIR + trap - SIGHUP + #trap - SIGCHLD + INSTOPSYSTEM=0 +} + +verify_prereq() { + # Verify Django version + reqfile=$(python -c "import os; print os.path.realpath('$BBBASEDIR/toaster-requirements.txt')") + exp='s/Django\([><=]\+\)\([^,]\+\),\([><=]\+\)\(.\+\)/' + exp=$exp'import sys,django;version=django.get_version().split(".");' + exp=$exp'sys.exit(not (version \1 "\2".split(".") and version \3 "\4".split(".")))/p' + if ! sed -n "$exp" $reqfile | python - ; then + req=`grep ^Django $reqfile` + echo "This program needs $req" + echo "Please install with pip install -r $reqfile" + return 2 + fi + + return 0 +} + +# read command line parameters +if [ -n "$BASH_SOURCE" ] ; then + TOASTER=${BASH_SOURCE} +elif [ -n "$ZSH_NAME" ] ; then + TOASTER=${(%):-%x} +else + TOASTER=$0 +fi + +export BBBASEDIR=`dirname $TOASTER`/.. +MANAGE=$BBBASEDIR/lib/toaster/manage.py +OEROOT=`dirname $TOASTER`/../.. + +# this is the configuraton file we are using for toaster +# we are using the same logic that oe-setup-builddir uses +# (based on TEMPLATECONF and .templateconf) to determine +# which toasterconf.json to use. +# note: There are a number of relative path assumptions +# in the local layers that currently make using an arbitrary +# toasterconf.json difficult. + +. $OEROOT/.templateconf +if [ -n "$TEMPLATECONF" ]; then + if [ ! -d "$TEMPLATECONF" ]; then + # Allow TEMPLATECONF=meta-xyz/conf as a shortcut + if [ -d "$OEROOT/$TEMPLATECONF" ]; then + TEMPLATECONF="$OEROOT/$TEMPLATECONF" + fi + if [ ! -d "$TEMPLATECONF" ]; then + echo >&2 "Error: '$TEMPLATECONF' must be a directory containing toasterconf.json" + return 1 + fi + fi +fi + +if [ "$TOASTER_CONF" = "" ]; then + TOASTER_CONF="$TEMPLATECONF/toasterconf.json" + export TOASTER_CONF=$(python -c "import os; print os.path.realpath('$TOASTER_CONF')") +fi + +if [ ! -f $TOASTER_CONF ]; then + echo "$TOASTER_CONF configuration file not found. Set TOASTER_CONF to specify file or fix .templateconf" + return 1 +fi + +# this defines the dir toaster will use for +# 1) clones of layers (in _toaster_clones ) +# 2) the build dir (in build) +# 3) the sqlite db if that is being used. +# 4) pid's we need to clean up on exit/shutdown +# note: for future. in order to make this an arbitrary directory, we need to +# make sure that the toaster.sqlite file doesn't default to `pwd` like it currently does. +export TOASTER_DIR=`pwd` + +WEBSERVER=1 +WEB_PORT="8000" +unset CMD +for param in $*; do + case $param in + noweb ) + WEBSERVER=0 + ;; + start ) + CMD=$param + ;; + stop ) + CMD=$param + ;; + webport=*) + WEB_PORT="${param#*=}" + esac +done + +if [ `basename \"$0\"` = `basename \"${TOASTER}\"` ]; then + echo "Error: This script needs to be sourced. Please run as . $TOASTER" + return 1 +fi + +verify_prereq || return 1 + +# We make sure we're running in the current shell and in a good environment +if [ -z "$BUILDDIR" ] || ! which bitbake >/dev/null 2>&1 ; then + echo "Error: Build environment is not setup or bitbake is not in path." 1>&2 + return 2 +fi + +# this defines the dir toaster will use for +# 1) clones of layers (in _toaster_clones ) +# 2) the build dir (in build) +# 3) the sqlite db if that is being used. +# 4) pid's we need to clean up on exit/shutdown +# note: for future. in order to make this an arbitrary directory, we need to +# make sure that the toaster.sqlite file doesn't default to `pwd` +# like it currently does. +export TOASTER_DIR=`dirname $BUILDDIR` + +# Determine the action. If specified by arguments, fine, if not, toggle it +if [ "$CMD" = "start" ] ; then + if [ -n "$BBSERVER" ]; then + echo " Toaster is already running. Exiting..." + return 1 +fi +elif [ "$CMD" = "" ]; then + if [ -z "$BBSERVER" ]; then + CMD="start" + else + CMD="stop" + fi +fi + +echo "The system will $CMD." + +# Execute the commands + +case $CMD in + start ) + # check if addr:port is not in use + if [ "$CMD" == 'start' ]; then + $MANAGE checksocket "0.0.0.0:$WEB_PORT" || return 1 + fi + + # kill Toaster web server if it's alive + if [ -e $BUILDDIR/.toastermain.pid ] && kill -0 `cat $BUILDDIR/.toastermain.pid`; then + echo "Warning: bitbake appears to be dead, but the Toaster web server is running." 1>&2 + echo " Something fishy is going on." 1>&2 + echo "Cleaning up the web server to start from a clean slate." + webserverKillAll + fi + + # Create configuration file + conf=${BUILDDIR}/conf/local.conf + line='INHERIT+="toaster buildhistory"' + grep -q "$line" $conf || echo $line >> $conf + + if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then + echo "Failed ${CMD}." + return 4 + fi + export BITBAKE_UI='toasterui' + export DATABASE_URL=`$MANAGE get-dburl` + $MANAGE runbuilds & echo $! >${BUILDDIR}/.runbuilds.pid + # set fail safe stop system on terminal exit + trap stop_system SIGHUP + echo "Successful ${CMD}." + return 0 + ;; + stop ) + stop_system + echo "Successful ${CMD}." + ;; +esac diff --git a/import-layers/yocto-poky/bitbake/bin/toaster-eventreplay b/import-layers/yocto-poky/bitbake/bin/toaster-eventreplay new file mode 100755 index 000000000..615a7aed1 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/bin/toaster-eventreplay @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2014 Alex Damian +# +# This file re-uses code spread throughout other Bitbake source files. +# As such, all other copyrights belong to their own right holders. +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# This command takes a filename as a single parameter. The filename is read +# as a build eventlog, and the ToasterUI is used to process events in the file +# and log data in the database + +from __future__ import print_function +import os +import sys, logging + +# mangle syspath to allow easy import of modules +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'lib')) + + +import bb.cooker +from bb.ui import toasterui +import sys +import logging + +import json, pickle + + +class FileReadEventsServerConnection(): + """ Emulates a connection to a bitbake server that feeds + events coming actually read from a saved log file. + """ + + class MockConnection(): + """ fill-in for the proxy to the server. we just return generic data + """ + def __init__(self, sc): + self._sc = sc + + def runCommand(self, commandArray): + """ emulates running a command on the server; only read-only commands are accepted """ + command_name = commandArray[0] + + if command_name == "getVariable": + if commandArray[1] in self._sc._variables: + return (self._sc._variables[commandArray[1]]['v'], None) + return (None, "Missing variable") + + elif command_name == "getAllKeysWithFlags": + dump = {} + flaglist = commandArray[1] + for k in self._sc._variables.keys(): + try: + if not k.startswith("__"): + v = self._sc._variables[k]['v'] + dump[k] = { + 'v' : v , + 'history' : self._sc._variables[k]['history'], + } + for d in flaglist: + dump[k][d] = self._sc._variables[k][d] + except Exception as e: + print(e) + return (dump, None) + else: + raise Exception("Command %s not implemented" % commandArray[0]) + + def terminateServer(self): + """ do not do anything """ + pass + + + + class EventReader(): + def __init__(self, sc): + self._sc = sc + self.firstraise = 0 + + def _create_event(self, line): + def _import_class(name): + assert len(name) > 0 + assert "." in name, name + + components = name.strip().split(".") + modulename = ".".join(components[:-1]) + moduleklass = components[-1] + + module = __import__(modulename, fromlist=[str(moduleklass)]) + return getattr(module, moduleklass) + + # we build a toaster event out of current event log line + try: + event_data = json.loads(line.strip()) + event_class = _import_class(event_data['class']) + event_object = pickle.loads(json.loads(event_data['vars'])) + except ValueError as e: + print("Failed loading ", line) + raise e + + if not isinstance(event_object, event_class): + raise Exception("Error loading objects %s class %s ", event_object, event_class) + + return event_object + + def waitEvent(self, timeout): + + nextline = self._sc._eventfile.readline() + if len(nextline) == 0: + # the build data ended, while toasterui still waits for events. + # this happens when the server was abruptly stopped, so we simulate this + self.firstraise += 1 + if self.firstraise == 1: + raise KeyboardInterrupt() + else: + return None + else: + self._sc.lineno += 1 + return self._create_event(nextline) + + + def _readVariables(self, variableline): + self._variables = json.loads(variableline.strip())['allvariables'] + + + def __init__(self, file_name): + self.connection = FileReadEventsServerConnection.MockConnection(self) + self._eventfile = open(file_name, "r") + + # we expect to have the variable dump at the start of the file + self.lineno = 1 + self._readVariables(self._eventfile.readline()) + + self.events = FileReadEventsServerConnection.EventReader(self) + + + + + +class MockConfigParameters(): + """ stand-in for cookerdata.ConfigParameters; as we don't really config a cooker, this + serves just to supply needed interfaces for the toaster ui to work """ + def __init__(self): + self.observe_only = True # we can only read files + + +# run toaster ui on our mock bitbake class +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: %s event.log " % sys.argv[0]) + sys.exit(1) + + file_name = sys.argv[-1] + mock_connection = FileReadEventsServerConnection(file_name) + configParams = MockConfigParameters() + + # run the main program and set exit code to the returned value + sys.exit(toasterui.main(mock_connection.connection, mock_connection.events, configParams)) diff --git a/import-layers/yocto-poky/bitbake/contrib/README b/import-layers/yocto-poky/bitbake/contrib/README new file mode 100644 index 000000000..25e515661 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/contrib/README @@ -0,0 +1 @@ +This directory is for additional contributed files which may be useful. diff --git a/import-layers/yocto-poky/bitbake/contrib/bbdev.sh b/import-layers/yocto-poky/bitbake/contrib/bbdev.sh new file mode 100644 index 000000000..33a78531e --- /dev/null +++ b/import-layers/yocto-poky/bitbake/contrib/bbdev.sh @@ -0,0 +1,31 @@ +# This is a shell function to be sourced into your shell or placed in your .profile, +# which makes setting things up for BitBake a bit easier. +# +# The author disclaims copyright to the contents of this file and places it in the +# public domain. + +bbdev () { + local BBDIR PKGDIR BUILDDIR + if test x"$1" = "x--help"; then echo >&2 "syntax: bbdev [bbdir [pkgdir [builddir]]]"; return 1; fi + if test x"$1" = x; then BBDIR=`pwd`; else BBDIR=$1; fi + if test x"$2" = x; then PKGDIR=`pwd`; else PKGDIR=$2; fi + if test x"$3" = x; then BUILDDIR=`pwd`; else BUILDDIR=$3; fi + + BBDIR=`readlink -f $BBDIR` + PKGDIR=`readlink -f $PKGDIR` + BUILDDIR=`readlink -f $BUILDDIR` + if ! (test -d $BBDIR && test -d $PKGDIR && test -d $BUILDDIR); then + echo >&2 "syntax: bbdev [bbdir [pkgdir [builddir]]]" + return 1 + fi + + PATH=$BBDIR/bin:$PATH + BBPATH=$BBDIR + if test x"$BBDIR" != x"$PKGDIR"; then + BBPATH=$PKGDIR:$BBPATH + fi + if test x"$PKGDIR" != x"$BUILDDIR"; then + BBPATH=$BUILDDIR:$BBPATH + fi + export BBPATH +} diff --git a/import-layers/yocto-poky/bitbake/contrib/dump_cache.py b/import-layers/yocto-poky/bitbake/contrib/dump_cache.py new file mode 100755 index 000000000..e1f23090b --- /dev/null +++ b/import-layers/yocto-poky/bitbake/contrib/dump_cache.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2012 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# +# This is used for dumping the bb_cache.dat, the output format is: +# recipe_path PN PV PACKAGES +# +import os +import sys +import warnings + +# For importing bb.cache +sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), '../lib')) +from bb.cache import CoreRecipeInfo + +import cPickle as pickle + +def main(argv=None): + """ + Get the mapping for the target recipe. + """ + if len(argv) != 1: + print >>sys.stderr, "Error, need one argument!" + return 2 + + cachefile = argv[0] + + with open(cachefile, "rb") as cachefile: + pickled = pickle.Unpickler(cachefile) + while cachefile: + try: + key = pickled.load() + val = pickled.load() + except Exception: + break + if isinstance(val, CoreRecipeInfo) and (not val.skipped): + pn = val.pn + # Filter out the native recipes. + if key.startswith('virtual:native:') or pn.endswith("-native"): + continue + + # 1.0 is the default version for a no PV recipe. + if val.__dict__.has_key("pv"): + pv = val.pv + else: + pv = "1.0" + + print("%s %s %s %s" % (key, pn, pv, ' '.join(val.packages))) + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) + diff --git a/import-layers/yocto-poky/bitbake/contrib/vim/ftdetect/bitbake.vim b/import-layers/yocto-poky/bitbake/contrib/vim/ftdetect/bitbake.vim new file mode 100644 index 000000000..200f8ae49 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/contrib/vim/ftdetect/bitbake.vim @@ -0,0 +1,24 @@ +" Vim filetype detection file +" Language: BitBake +" Author: Ricardo Salveti +" Copyright: Copyright (C) 2008 Ricardo Salveti +" Licence: You may redistribute this under the same terms as Vim itself +" +" This sets up the syntax highlighting for BitBake files, like .bb, .bbclass and .inc + +if &compatible || version < 600 + finish +endif + +" .bb, .bbappend and .bbclass +au BufNewFile,BufRead *.{bb,bbappend,bbclass} set filetype=bitbake + +" .inc +au BufNewFile,BufRead *.inc set filetype=bitbake + +" .conf +au BufNewFile,BufRead *.conf + \ if (match(expand("%:p:h"), "conf") > 0) | + \ set filetype=bitbake | + \ endif + diff --git a/import-layers/yocto-poky/bitbake/contrib/vim/ftplugin/bitbake.vim b/import-layers/yocto-poky/bitbake/contrib/vim/ftplugin/bitbake.vim new file mode 100644 index 000000000..db0d75319 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/contrib/vim/ftplugin/bitbake.vim @@ -0,0 +1,2 @@ +set sts=4 sw=4 et +set cms=#%s diff --git a/import-layers/yocto-poky/bitbake/contrib/vim/plugin/newbb.vim b/import-layers/yocto-poky/bitbake/contrib/vim/plugin/newbb.vim new file mode 100755 index 000000000..874e33805 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/contrib/vim/plugin/newbb.vim @@ -0,0 +1,84 @@ +" Vim plugin file +" Purpose: Create a template for new bb files +" Author: Ricardo Salveti +" Copyright: Copyright (C) 2008 Ricardo Salveti +" +" This file is licensed under the MIT license, see COPYING.MIT in +" this source distribution for the terms. +" +" Based on the gentoo-syntax package +" +" Will try to use git to find the user name and email + +if &compatible || v:version < 600 + finish +endif + +fun! GetUserName() + let l:user_name = system("git config --get user.name") + if v:shell_error + return "Unknown User" + else + return substitute(l:user_name, "\n", "", "") +endfun + +fun! GetUserEmail() + let l:user_email = system("git config --get user.email") + if v:shell_error + return "unknow@user.org" + else + return substitute(l:user_email, "\n", "", "") +endfun + +fun! BBHeader() + let l:current_year = strftime("%Y") + let l:user_name = GetUserName() + let l:user_email = GetUserEmail() + 0 put ='# Copyright (C) ' . l:current_year . + \ ' ' . l:user_name . ' <' . l:user_email . '>' + put ='# Released under the MIT license (see COPYING.MIT for the terms)' + $ +endfun + +fun! NewBBTemplate() + let l:paste = &paste + set nopaste + + " Get the header + call BBHeader() + + " New the bb template + put ='DESCRIPTION = \"\"' + put ='HOMEPAGE = \"\"' + put ='LICENSE = \"\"' + put ='SECTION = \"\"' + put ='DEPENDS = \"\"' + put ='' + put ='SRC_URI = \"\"' + + " Go to the first place to edit + 0 + /^DESCRIPTION =/ + exec "normal 2f\"" + + if paste == 1 + set paste + endif +endfun + +if !exists("g:bb_create_on_empty") + let g:bb_create_on_empty = 1 +endif + +" disable in case of vimdiff +if v:progname =~ "vimdiff" + let g:bb_create_on_empty = 0 +endif + +augroup NewBB + au BufNewFile *.bb + \ if g:bb_create_on_empty | + \ call NewBBTemplate() | + \ endif +augroup END + diff --git a/import-layers/yocto-poky/bitbake/contrib/vim/syntax/bitbake.vim b/import-layers/yocto-poky/bitbake/contrib/vim/syntax/bitbake.vim new file mode 100644 index 000000000..fb55f9102 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/contrib/vim/syntax/bitbake.vim @@ -0,0 +1,126 @@ +" Vim syntax file +" Language: BitBake bb/bbclasses/inc +" Author: Chris Larson +" Ricardo Salveti +" Copyright: Copyright (C) 2004 Chris Larson +" Copyright (C) 2008 Ricardo Salveti +" +" This file is licensed under the MIT license, see COPYING.MIT in +" this source distribution for the terms. +" +" Syntax highlighting for bb, bbclasses and inc files. +" +" It's an entirely new type, just has specific syntax in shell and python code + +if &compatible || v:version < 600 + finish +endif +if exists("b:current_syntax") + finish +endif + +syn include @python syntax/python.vim +if exists("b:current_syntax") + unlet b:current_syntax +endif + +" BitBake syntax + +" Matching case +syn case match + +" Indicates the error when nothing is matched +syn match bbUnmatched "." + +" Comments +syn cluster bbCommentGroup contains=bbTodo,@Spell +syn keyword bbTodo COMBAK FIXME TODO XXX contained +syn match bbComment "#.*$" contains=@bbCommentGroup + +" String helpers +syn match bbQuote +['"]+ contained +syn match bbDelimiter "[(){}=]" contained +syn match bbArrayBrackets "[\[\]]" contained + +" BitBake strings +syn match bbContinue "\\$" +syn region bbString matchgroup=bbQuote start=+"+ skip=+\\$+ end=+"+ contained contains=bbTodo,bbContinue,bbVarDeref,bbVarPyValue,@Spell +syn region bbString matchgroup=bbQuote start=+'+ skip=+\\$+ end=+'+ contained contains=bbTodo,bbContinue,bbVarDeref,bbVarPyValue,@Spell + +" Vars definition +syn match bbExport "^export" nextgroup=bbIdentifier skipwhite +syn keyword bbExportFlag export contained nextgroup=bbIdentifier skipwhite +syn match bbIdentifier "[a-zA-Z0-9\-_\.\/\+]\+" display contained +syn match bbVarDeref "${[a-zA-Z0-9\-_\.\/\+]\+}" contained +syn match bbVarEq "\(:=\|+=\|=+\|\.=\|=\.\|?=\|??=\|=\)" contained nextgroup=bbVarValue +syn match bbVarDef "^\(export\s*\)\?\([a-zA-Z0-9\-_\.\/\+]\+\(_[${}a-zA-Z0-9\-_\.\/\+]\+\)\?\)\s*\(:=\|+=\|=+\|\.=\|=\.\|?=\|??=\|=\)\@=" contains=bbExportFlag,bbIdentifier,bbVarDeref nextgroup=bbVarEq +syn match bbVarValue ".*$" contained contains=bbString,bbVarDeref,bbVarPyValue +syn region bbVarPyValue start=+${@+ skip=+\\$+ end=+}+ contained contains=@python + +" Vars metadata flags +syn match bbVarFlagDef "^\([a-zA-Z0-9\-_\.]\+\)\(\[[a-zA-Z0-9\-_\.]\+\]\)\@=" contains=bbIdentifier nextgroup=bbVarFlagFlag +syn region bbVarFlagFlag matchgroup=bbArrayBrackets start="\[" end="\]\s*\(=\|+=\|=+\|?=\)\@=" contained contains=bbIdentifier nextgroup=bbVarEq + +" Includes and requires +syn keyword bbInclude inherit include require contained +syn match bbIncludeRest ".*$" contained contains=bbString,bbVarDeref +syn match bbIncludeLine "^\(inherit\|include\|require\)\s\+" contains=bbInclude nextgroup=bbIncludeRest + +" Add taks and similar +syn keyword bbStatement addtask addhandler after before EXPORT_FUNCTIONS contained +syn match bbStatementRest ".*$" skipwhite contained contains=bbStatement +syn match bbStatementLine "^\(addtask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest + +" OE Important Functions +syn keyword bbOEFunctions do_fetch do_unpack do_patch do_configure do_compile do_stage do_install do_package contained + +" Generic Functions +syn match bbFunction "\h[0-9A-Za-z_-]*" display contained contains=bbOEFunctions + +" BitBake shell metadata +syn include @shell syntax/sh.vim +if exists("b:current_syntax") + unlet b:current_syntax +endif +syn keyword bbShFakeRootFlag fakeroot contained +syn match bbShFuncDef "^\(fakeroot\s*\)\?\([0-9A-Za-z_${}-]\+\)\(python\)\@ + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/import-layers/yocto-poky/bitbake/doc/COPYING.MIT b/import-layers/yocto-poky/bitbake/doc/COPYING.MIT new file mode 100644 index 000000000..7e7d57413 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/COPYING.MIT @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/import-layers/yocto-poky/bitbake/doc/Makefile b/import-layers/yocto-poky/bitbake/doc/Makefile new file mode 100644 index 000000000..3c28f4b22 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/Makefile @@ -0,0 +1,91 @@ +# This is a single Makefile to handle all generated BitBake documents. +# The Makefile needs to live in the documentation directory and all figures used +# in any manuals must be .PNG files and live in the individual book's figures +# directory. +# +# The Makefile has these targets: +# +# pdf: generates a PDF version of a manual. +# html: generates an HTML version of a manual. +# tarball: creates a tarball for the doc files. +# validate: validates +# clean: removes files +# +# The Makefile generates an HTML version of every document. The +# variable DOC indicates the folder name for a given manual. +# +# To build a manual, you must invoke 'make' with the DOC argument. +# +# Examples: +# +# make DOC=bitbake-user-manual +# make pdf DOC=bitbake-user-manual +# +# The first example generates the HTML version of the User Manual. +# The second example generates the PDF version of the User Manual. +# + +ifeq ($(DOC),bitbake-user-manual) +XSLTOPTS = --stringparam html.stylesheet bitbake-user-manual-style.css \ + --stringparam chapter.autolabel 1 \ + --stringparam section.autolabel 1 \ + --stringparam section.label.includes.component.label 1 \ + --xinclude +ALLPREQ = html tarball +TARFILES = bitbake-user-manual-style.css bitbake-user-manual.html figures/bitbake-title.png +MANUALS = $(DOC)/$(DOC).html +FIGURES = figures +STYLESHEET = $(DOC)/*.css + +endif + +## +# These URI should be rewritten by your distribution's xml catalog to +# match your localy installed XSL stylesheets. +XSL_BASE_URI = http://docbook.sourceforge.net/release/xsl/current +XSL_XHTML_URI = $(XSL_BASE_URI)/xhtml/docbook.xsl + +all: $(ALLPREQ) + +pdf: +ifeq ($(DOC),bitbake-user-manual) + @echo " " + @echo "********** Building."$(DOC) + @echo " " + cd $(DOC); ../tools/docbook-to-pdf $(DOC).xml ../template; cd .. +endif + +html: +ifeq ($(DOC),bitbake-user-manual) +# See http://www.sagehill.net/docbookxsl/HtmlOutput.html + @echo " " + @echo "******** Building "$(DOC) + @echo " " + cd $(DOC); xsltproc $(XSLTOPTS) -o $(DOC).html $(DOC)-customization.xsl $(DOC).xml; cd .. +endif + +tarball: html + @echo " " + @echo "******** Creating Tarball of document files" + @echo " " + cd $(DOC); tar -cvzf $(DOC).tgz $(TARFILES); cd .. + +validate: + cd $(DOC); xmllint --postvalid --xinclude --noout $(DOC).xml; cd .. + +publish: + @if test -f $(DOC)/$(DOC).html; \ + then \ + echo " "; \ + echo "******** Publishing "$(DOC)".html"; \ + echo " "; \ + scp -r $(MANUALS) $(STYLESHEET) docs.yp:/var/www/www.yoctoproject.org-docs/$(VER)/$(DOC); \ + cd $(DOC); scp -r $(FIGURES) docs.yp:/var/www/www.yoctoproject.org-docs/$(VER)/$(DOC); \ + else \ + echo " "; \ + echo $(DOC)".html missing. Generate the file first then try again."; \ + echo " "; \ + fi + +clean: + rm -rf $(MANUALS); rm $(DOC)/$(DOC).tgz; diff --git a/import-layers/yocto-poky/bitbake/doc/README b/import-layers/yocto-poky/bitbake/doc/README new file mode 100644 index 000000000..303cf8eec --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/README @@ -0,0 +1,39 @@ +Documentation +============= + +This is the directory that contains the BitBake documentation. + +Manual Organization +=================== + +Folders exist for individual manuals as follows: + +* bitbake-user-manual - The BitBake User Manual + +Each folder is self-contained regarding content and figures. + +If you want to find HTML versions of the BitBake manuals on the web, +go to http://www.openembedded.org/wiki/Documentation. + +Makefile +======== + +The Makefile processes manual directories to create HTML, PDF, +tarballs, etc. Details on how the Makefile work are documented +inside the Makefile. See that file for more information. + +To build a manual, you run the make command and pass it the name +of the folder containing the manual's contents. +For example, the following command run from the documentation directory +creates an HTML and a PDF version of the BitBake User Manual. +The DOC variable specifies the manual you are making: + + $ make DOC=bitbake-user-manual + +template +======== +Contains various templates, fonts, and some old PNG files. + +tools +===== +Contains a tool to convert the DocBook files to PDF format. diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-customization.xsl b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-customization.xsl new file mode 100644 index 000000000..5985ea783 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-customization.xsl @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + A + + + + diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml new file mode 100644 index 000000000..b1b72e0aa --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml @@ -0,0 +1,931 @@ + + + + Execution + + + The primary purpose for running BitBake is to produce some kind + of output such as a single installable package, a kernel, a software + development kit, or even a full, board-specific bootable Linux image, + complete with bootloader, kernel, and root filesystem. + Of course, you can execute the bitbake + command with options that cause it to execute single tasks, + compile single recipe files, capture or clear data, or simply + return information about the execution environment. + + + + This chapter describes BitBake's execution process from start + to finish when you use it to create an image. + The execution process is launched using the following command + form: + + $ bitbake target + + For information on the BitBake command and its options, + see + "The BitBake Command" + section. + + + Prior to executing BitBake, you should take advantage of available + parallel thread execution on your build host by setting the + BB_NUMBER_THREADS + variable in your project's local.conf + configuration file. + + + + A common method to determine this value for your build host is to run + the following: + + $ grep processor /proc/cpuinfo + + This command returns the number of processors, which takes into + account hyper-threading. + Thus, a quad-core build host with hyper-threading most likely + shows eight processors, which is the value you would then assign to + BB_NUMBER_THREADS. + + + + A possibly simpler solution is that some Linux distributions + (e.g. Debian and Ubuntu) provide the ncpus command. + + + + +
+ Parsing the Base Configuration Metadata + + + The first thing BitBake does is parse base configuration + metadata. + Base configuration metadata consists of your project's + bblayers.conf file to determine what + layers BitBake needs to recognize, all necessary + layer.conf files (one from each layer), + and bitbake.conf. + The data itself is of various types: + + Recipes: + Details about particular pieces of software. + + Class Data: + An abstraction of common build information + (e.g. how to build a Linux kernel). + + Configuration Data: + Machine-specific settings, policy decisions, + and so forth. + Configuration data acts as the glue to bind everything + together. + + + + + The layer.conf files are used to + construct key variables such as + BBPATH + and + BBFILES. + BBPATH is used to search for + configuration and class files under the + conf and classes + directories, respectively. + BBFILES is used to locate both recipe + and recipe append files + (.bb and .bbappend). + If there is no bblayers.conf file, + it is assumed the user has set the BBPATH + and BBFILES directly in the environment. + + + + Next, the bitbake.conf file is located + using the BBPATH variable that was + just constructed. + The bitbake.conf file may also include other + configuration files using the + include or + require directives. + + + + Prior to parsing configuration files, Bitbake looks + at certain variables, including: + + + BB_ENV_WHITELIST + + + BB_ENV_EXTRAWHITE + + + BB_PRESERVE_ENV + + + BB_ORIGENV + + + BITBAKE_UI + + + The first four variables in this list relate to how BitBake treats shell + environment variables during task execution. + By default, BitBake cleans the environment variables and provides tight + control over the shell execution environment. + However, through the use of these first four variables, you can + apply your control regarding the + environment variables allowed to be used by BitBake in the shell + during execution of tasks. + See the + "Passing Information Into the Build Task Environment" + section and the information about these variables in the + variable glossary for more information on how they work and + on how to use them. + + + + The base configuration metadata is global + and therefore affects all recipes and tasks that are executed. + + + + BitBake first searches the current working directory for an + optional conf/bblayers.conf configuration file. + This file is expected to contain a + BBLAYERS + variable that is a space-delimited list of 'layer' directories. + Recall that if BitBake cannot find a bblayers.conf + file, then it is assumed the user has set the BBPATH + and BBFILES variables directly in the environment. + + + + For each directory (layer) in this list, a conf/layer.conf + file is located and parsed with the + LAYERDIR + variable being set to the directory where the layer was found. + The idea is these files automatically set up + BBPATH + and other variables correctly for a given build directory. + + + + BitBake then expects to find the conf/bitbake.conf + file somewhere in the user-specified BBPATH. + That configuration file generally has include directives to pull + in any other metadata such as files specific to the architecture, + the machine, the local environment, and so forth. + + + + Only variable definitions and include directives are allowed + in BitBake .conf files. + Some variables directly influence BitBake's behavior. + These variables might have been set from the environment + depending on the environment variables previously + mentioned or set in the configuration files. + The + "Variables Glossary" + chapter presents a full list of variables. + + + + After parsing configuration files, BitBake uses its rudimentary + inheritance mechanism, which is through class files, to inherit + some standard classes. + BitBake parses a class when the inherit directive responsible + for getting that class is encountered. + + + + The base.bbclass file is always included. + Other classes that are specified in the configuration using the + INHERIT + variable are also included. + BitBake searches for class files in a + classes subdirectory under + the paths in BBPATH in the same way as + configuration files. + + + + A good way to get an idea of the configuration files and + the class files used in your execution environment is to + run the following BitBake command: + + $ bitbake -e > mybb.log + + Examining the top of the mybb.log + shows you the many configuration files and class files + used in your execution environment. + + + + + You need to be aware of how BitBake parses curly braces. + If a recipe uses a closing curly brace within the function and + the character has no leading spaces, BitBake produces a parsing + error. + If you use a pair of curly braces in a shell function, the + closing curly brace must not be located at the start of the line + without leading spaces. + + + + Here is an example that causes BitBake to produce a parsing + error: + + fakeroot create_shar() { + cat << "EOF" > ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh + usage() + { + echo "test" + ###### The following "}" at the start of the line causes a parsing error ###### + } + EOF + } + + Writing the recipe this way avoids the error: + + fakeroot create_shar() { + cat << "EOF" > ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh + usage() + { + echo "test" + ######The following "}" with a leading space at the start of the line avoids the error ###### + } + EOF + } + + + +
+ +
+ Locating and Parsing Recipes + + + During the configuration phase, BitBake will have set + BBFILES. + BitBake now uses it to construct a list of recipes to parse, + along with any append files (.bbappend) + to apply. + BBFILES is a space-separated list of + available files and supports wildcards. + An example would be: + + BBFILES = "/path/to/bbfiles/*.bb /path/to/appends/*.bbappend" + + BitBake parses each recipe and append file located + with BBFILES and stores the values of + various variables into the datastore. + + Append files are applied in the order they are encountered in + BBFILES. + + For each file, a fresh copy of the base configuration is + made, then the recipe is parsed line by line. + Any inherit statements cause BitBake to find and + then parse class files (.bbclass) + using + BBPATH + as the search path. + Finally, BitBake parses in order any append files found in + BBFILES. + + + + One common convention is to use the recipe filename to define + pieces of metadata. + For example, in bitbake.conf the recipe + name and version are used to set the variables + PN and + PV: + + PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" + PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}" + + In this example, a recipe called "something_1.2.3.bb" would set + PN to "something" and + PV to "1.2.3". + + + + By the time parsing is complete for a recipe, BitBake + has a list of tasks that the recipe defines and a set of + data consisting of keys and values as well as + dependency information about the tasks. + + + + BitBake does not need all of this information. + It only needs a small subset of the information to make + decisions about the recipe. + Consequently, BitBake caches the values in which it is + interested and does not store the rest of the information. + Experience has shown it is faster to re-parse the metadata than to + try and write it out to the disk and then reload it. + + + + Where possible, subsequent BitBake commands reuse this cache of + recipe information. + The validity of this cache is determined by first computing a + checksum of the base configuration data (see + BB_HASHCONFIG_WHITELIST) + and then checking if the checksum matches. + If that checksum matches what is in the cache and the recipe + and class files have not changed, Bitbake is able to use + the cache. + BitBake then reloads the cached information about the recipe + instead of reparsing it from scratch. + + + + Recipe file collections exist to allow the user to + have multiple repositories of + .bb files that contain the same + exact package. + For example, one could easily use them to make one's + own local copy of an upstream repository, but with + custom modifications that one does not want upstream. + Here is an example: + + BBFILES = "/stuff/openembedded/*/*.bb /stuff/openembedded.modified/*/*.bb" + BBFILE_COLLECTIONS = "upstream local" + BBFILE_PATTERN_upstream = "^/stuff/openembedded/" + BBFILE_PATTERN_local = "^/stuff/openembedded.modified/" + BBFILE_PRIORITY_upstream = "5" + BBFILE_PRIORITY_local = "10" + + + The layers mechanism is now the preferred method of collecting + code. + While the collections code remains, its main use is to set layer + priorities and to deal with overlap (conflicts) between layers. + + +
+ +
+ Providers + + + Assuming BitBake has been instructed to execute a target + and that all the recipe files have been parsed, BitBake + starts to figure out how to build the target. + BitBake looks through the PROVIDES list + for each of the recipes. + A PROVIDES list is the list of names by which + the recipe can be known. + Each recipe's PROVIDES list is created + implicitly through the recipe's + PN variable + and explicitly through the recipe's + PROVIDES + variable, which is optional. + + + + When a recipe uses PROVIDES, that recipe's + functionality can be found under an alternative name or names other + than the implicit PN name. + As an example, suppose a recipe named keyboard_1.0.bb + contained the following: + + PROVIDES += "fullkeyboard" + + The PROVIDES list for this recipe becomes + "keyboard", which is implicit, and "fullkeyboard", which is explicit. + Consequently, the functionality found in + keyboard_1.0.bb can be found under two + different names. + +
+ +
+ Preferences + + + The PROVIDES list is only part of the solution + for figuring out a target's recipes. + Because targets might have multiple providers, BitBake needs + to prioritize providers by determining provider preferences. + + + + A common example in which a target has multiple providers + is "virtual/kernel", which is on the + PROVIDES list for each kernel recipe. + Each machine often selects the best kernel provider by using a + line similar to the following in the machine configuration file: + + PREFERRED_PROVIDER_virtual/kernel = "linux-yocto" + + The default + PREFERRED_PROVIDER + is the provider with the same name as the target. + Bitbake iterates through each target it needs to build and + resolves them and their dependencies using this process. + + + + Understanding how providers are chosen is made complicated by the fact + that multiple versions might exist for a given provider. + BitBake defaults to the highest version of a provider. + Version comparisons are made using the same method as Debian. + You can use the + PREFERRED_VERSION + variable to specify a particular version. + You can influence the order by using the + DEFAULT_PREFERENCE + variable. + + + + By default, files have a preference of "0". + Setting DEFAULT_PREFERENCE to "-1" makes the + recipe unlikely to be used unless it is explicitly referenced. + Setting DEFAULT_PREFERENCE to "1" makes it + likely the recipe is used. + PREFERRED_VERSION overrides any + DEFAULT_PREFERENCE setting. + DEFAULT_PREFERENCE is often used to mark newer + and more experimental recipe versions until they have undergone + sufficient testing to be considered stable. + + + + When there are multiple “versions†of a given recipe, + BitBake defaults to selecting the most recent + version, unless otherwise specified. + If the recipe in question has a + DEFAULT_PREFERENCE + set lower than the other recipes (default is 0), then + it will not be selected. + This allows the person or persons maintaining + the repository of recipe files to specify + their preference for the default selected version. + Additionally, the user can specify their preferred version. + + + + If the first recipe is named a_1.1.bb, then the + PN variable + will be set to “aâ€, and the + PV + variable will be set to 1.1. + + + + Thus, if a recipe named a_1.2.bb exists, BitBake + will choose 1.2 by default. + However, if you define the following variable in a + .conf file that BitBake parses, you + can change that preference: + + PREFERRED_VERSION_a = "1.1" + + + + + + It is common for a recipe to provide two versions -- a stable, + numbered (and preferred) version, and a version that is + automatically checked out from a source code repository that + is considered more "bleeding edge" but can be selected only + explicitly. + + + + For example, in the OpenEmbedded codebase, there is a standard, + versioned recipe file for BusyBox, + busybox_1.22.1.bb, + but there is also a Git-based version, + busybox_git.bb, which explicitly contains the line + + DEFAULT_PREFERENCE = "-1" + + to ensure that the numbered, stable version is always preferred + unless the developer selects otherwise. + + +
+ +
+ Dependencies + + + Each target BitBake builds consists of multiple tasks such as + fetch, unpack, + patch, configure, + and compile. + For best performance on multi-core systems, BitBake considers each + task as an independent + entity with its own set of dependencies. + + + + Dependencies are defined through several variables. + You can find information about variables BitBake uses in + the Variables Glossary + near the end of this manual. + At a basic level, it is sufficient to know that BitBake uses the + DEPENDS and + RDEPENDS variables when + calculating dependencies. + + + + For more information on how BitBake handles dependencies, see the + "Dependencies" section. + +
+ +
+ The Task List + + + Based on the generated list of providers and the dependency information, + BitBake can now calculate exactly what tasks it needs to run and in what + order it needs to run them. + The + "Executing Tasks" section has more + information on how BitBake chooses which task to execute next. + + + + The build now starts with BitBake forking off threads up to the limit set in the + BB_NUMBER_THREADS + variable. + BitBake continues to fork threads as long as there are tasks ready to run, + those tasks have all their dependencies met, and the thread threshold has not been + exceeded. + + + + It is worth noting that you can greatly speed up the build time by properly setting + the BB_NUMBER_THREADS variable. + + + + As each task completes, a timestamp is written to the directory specified by the + STAMP variable. + On subsequent runs, BitBake looks in the build directory within + tmp/stamps and does not rerun + tasks that are already completed unless a timestamp is found to be invalid. + Currently, invalid timestamps are only considered on a per + recipe file basis. + So, for example, if the configure stamp has a timestamp greater than the + compile timestamp for a given target, then the compile task would rerun. + Running the compile task again, however, has no effect on other providers + that depend on that target. + + + + The exact format of the stamps is partly configurable. + In modern versions of BitBake, a hash is appended to the + stamp so that if the configuration changes, the stamp becomes + invalid and the task is automatically rerun. + This hash, or signature used, is governed by the signature policy + that is configured (see the + "Checksums (Signatures)" + section for information). + It is also possible to append extra metadata to the stamp using + the "stamp-extra-info" task flag. + For example, OpenEmbedded uses this flag to make some tasks machine-specific. + + + + Some tasks are marked as "nostamp" tasks. + No timestamp file is created when these tasks are run. + Consequently, "nostamp" tasks are always rerun. + + + + For more information on tasks, see the + "Tasks" section. + +
+ +
+ Executing Tasks + + + Tasks can be either a shell task or a Python task. + For shell tasks, BitBake writes a shell script to + ${T}/run.do_taskname.pid + and then executes the script. + The generated shell script contains all the exported variables, + and the shell functions with all variables expanded. + Output from the shell script goes to the file + ${T}/log.do_taskname.pid. + Looking at the expanded shell functions in the run file and + the output in the log files is a useful debugging technique. + + + + For Python tasks, BitBake executes the task internally and logs + information to the controlling terminal. + Future versions of BitBake will write the functions to files + similar to the way shell tasks are handled. + Logging will be handled in a way similar to shell tasks as well. + + + + The order in which BitBake runs the tasks is controlled by its + task scheduler. + It is possible to configure the scheduler and define custom + implementations for specific use cases. + For more information, see these variables that control the + behavior: + + + BB_SCHEDULER + + + BB_SCHEDULERS + + + It is possible to have functions run before and after a task's main + function. + This is done using the "prefuncs" and "postfuncs" flags of the task + that lists the functions to run. + +
+ +
+ Checksums (Signatures) + + + A checksum is a unique signature of a task's inputs. + The signature of a task can be used to determine if a task + needs to be run. + Because it is a change in a task's inputs that triggers running + the task, BitBake needs to detect all the inputs to a given task. + For shell tasks, this turns out to be fairly easy because + BitBake generates a "run" shell script for each task and + it is possible to create a checksum that gives you a good idea of when + the task's data changes. + + + + To complicate the problem, some things should not be included in + the checksum. + First, there is the actual specific build path of a given task - + the working directory. + It does not matter if the working directory changes because it should not + affect the output for target packages. + The simplistic approach for excluding the working directory is to set + it to some fixed value and create the checksum for the "run" script. + BitBake goes one step better and uses the + BB_HASHBASE_WHITELIST + variable to define a list of variables that should never be included + when generating the signatures. + + + + Another problem results from the "run" scripts containing functions that + might or might not get called. + The incremental build solution contains code that figures out dependencies + between shell functions. + This code is used to prune the "run" scripts down to the minimum set, + thereby alleviating this problem and making the "run" scripts much more + readable as a bonus. + + + + So far we have solutions for shell scripts. + What about Python tasks? + The same approach applies even though these tasks are more difficult. + The process needs to figure out what variables a Python function accesses + and what functions it calls. + Again, the incremental build solution contains code that first figures out + the variable and function dependencies, and then creates a checksum for the data + used as the input to the task. + + + + Like the working directory case, situations exist where dependencies + should be ignored. + For these cases, you can instruct the build process to ignore a dependency + by using a line like the following: + + PACKAGE_ARCHS[vardepsexclude] = "MACHINE" + + This example ensures that the PACKAGE_ARCHS variable does not + depend on the value of MACHINE, even if it does reference it. + + + + Equally, there are cases where we need to add dependencies BitBake + is not able to find. + You can accomplish this by using a line like the following: + + PACKAGE_ARCHS[vardeps] = "MACHINE" + + This example explicitly adds the MACHINE variable as a + dependency for PACKAGE_ARCHS. + + + + Consider a case with in-line Python, for example, where BitBake is not + able to figure out dependencies. + When running in debug mode (i.e. using -DDD), BitBake + produces output when it discovers something for which it cannot figure out + dependencies. + + + + Thus far, this section has limited discussion to the direct inputs into a task. + Information based on direct inputs is referred to as the "basehash" in the + code. + However, there is still the question of a task's indirect inputs - the + things that were already built and present in the build directory. + The checksum (or signature) for a particular task needs to add the hashes + of all the tasks on which the particular task depends. + Choosing which dependencies to add is a policy decision. + However, the effect is to generate a master checksum that combines the basehash + and the hashes of the task's dependencies. + + + + At the code level, there are a variety of ways both the basehash and the + dependent task hashes can be influenced. + Within the BitBake configuration file, we can give BitBake some extra information + to help it construct the basehash. + The following statement effectively results in a list of global variable + dependency excludes - variables never included in any checksum. + This example uses variables from OpenEmbedded to help illustrate + the concept: + + BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \ + SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \ + USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \ + PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \ + CCACHE_DIR EXTERNAL_TOOLCHAIN CCACHE CCACHE_DISABLE LICENSE_PATH SDKPKGSUFFIX" + + The previous example excludes the work directory, which is part of + TMPDIR. + + + + The rules for deciding which hashes of dependent tasks to include through + dependency chains are more complex and are generally accomplished with a + Python function. + The code in meta/lib/oe/sstatesig.py shows two examples + of this and also illustrates how you can insert your own policy into the system + if so desired. + This file defines the two basic signature generators OpenEmbedded Core + uses: "OEBasic" and "OEBasicHash". + By default, there is a dummy "noop" signature handler enabled in BitBake. + This means that behavior is unchanged from previous versions. + OE-Core uses the "OEBasicHash" signature handler by default + through this setting in the bitbake.conf file: + + BB_SIGNATURE_HANDLER ?= "OEBasicHash" + + The "OEBasicHash" BB_SIGNATURE_HANDLER is the same as the + "OEBasic" version but adds the task hash to the stamp files. + This results in any metadata change that changes the task hash, automatically + causing the task to be run again. + This removes the need to bump + PR + values, and changes to metadata automatically ripple across the build. + + + + It is also worth noting that the end result of these signature generators is to + make some dependency and hash information available to the build. + This information includes: + + BB_BASEHASH_task-taskname: + The base hashes for each task in the recipe. + + BB_BASEHASH_filename:taskname: + The base hashes for each dependent task. + + BBHASHDEPS_filename:taskname: + The task dependencies for each task. + + BB_TASKHASH: + The hash of the currently running task. + + + + + + It is worth noting that BitBake's "-S" option lets you + debug Bitbake's processing of signatures. + The options passed to -S allow different debugging modes + to be used, either using BitBake's own debug functions + or possibly those defined in the metadata/signature handler + itself. + The simplest parameter to pass is "none", which causes a + set of signature information to be written out into + STAMP_DIR + corresponding to the targets specified. + The other currently available parameter is "printdiff", + which causes BitBake to try to establish the closest + signature match it can (e.g. in the sstate cache) and then + run bitbake-diffsigs over the matches + to determine the stamps and delta where these two + stamp trees diverge. + + It is likely that future versions of BitBake will + provide other signature handlers triggered through + additional "-S" parameters. + + + + + You can find more information on checksum metadata in the + "Task Checksums and Setscene" + section. + +
+ +
+ Setscene + + + The setscene process enables BitBake to handle "pre-built" artifacts. + The ability to handle and reuse these artifacts allows BitBake + the luxury of not having to build something from scratch every time. + Instead, BitBake can use, when possible, existing build artifacts. + + + + BitBake needs to have reliable data indicating whether or not an + artifact is compatible. + Signatures, described in the previous section, provide an ideal + way of representing whether an artifact is compatible. + If a signature is the same, an object can be reused. + + + + If an object can be reused, the problem then becomes how to + replace a given task or set of tasks with the pre-built artifact. + BitBake solves the problem with the "setscene" process. + + + + When BitBake is asked to build a given target, before building anything, + it first asks whether cached information is available for any of the + targets it's building, or any of the intermediate targets. + If cached information is available, BitBake uses this information instead of + running the main tasks. + + + + BitBake first calls the function defined by the + BB_HASHCHECK_FUNCTION + variable with a list of tasks and corresponding + hashes it wants to build. + This function is designed to be fast and returns a list + of the tasks for which it believes in can obtain artifacts. + + + + Next, for each of the tasks that were returned as possibilities, + BitBake executes a setscene version of the task that the possible + artifact covers. + Setscene versions of a task have the string "_setscene" appended to the + task name. + So, for example, the task with the name xxx has + a setscene task named xxx_setscene. + The setscene version of the task executes and provides the necessary + artifacts returning either success or failure. + + + + As previously mentioned, an artifact can cover more than one task. + For example, it is pointless to obtain a compiler if you + already have the compiled binary. + To handle this, BitBake calls the + BB_SETSCENE_DEPVALID + function for each successful setscene task to know whether or not it needs + to obtain the dependencies of that task. + + + + Finally, after all the setscene tasks have executed, BitBake calls the + function listed in + BB_SETSCENE_VERIFY_FUNCTION + with the list of tasks BitBake thinks has been "covered". + The metadata can then ensure that this list is correct and can + inform BitBake that it wants specific tasks to be run regardless + of the setscene result. + + + + You can find more information on setscene metadata in the + "Task Checksums and Setscene" + section. + +
+ diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml new file mode 100644 index 000000000..f168cfa68 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml @@ -0,0 +1,765 @@ + + + +File Download Support + + + BitBake's fetch module is a standalone piece of library code + that deals with the intricacies of downloading source code + and files from remote systems. + Fetching source code is one of the cornerstones of building software. + As such, this module forms an important part of BitBake. + + + + The current fetch module is called "fetch2" and refers to the + fact that it is the second major version of the API. + The original version is obsolete and has been removed from the codebase. + Thus, in all cases, "fetch" refers to "fetch2" in this + manual. + + +
+ The Download (Fetch) + + + BitBake takes several steps when fetching source code or files. + The fetcher codebase deals with two distinct processes in order: + obtaining the files from somewhere (cached or otherwise) + and then unpacking those files into a specific location and + perhaps in a specific way. + Getting and unpacking the files is often optionally followed + by patching. + Patching, however, is not covered by this module. + + + + The code to execute the first part of this process, a fetch, + looks something like the following: + + src_uri = (d.getVar('SRC_URI', True) or "").split() + fetcher = bb.fetch2.Fetch(src_uri, d) + fetcher.download() + + This code sets up an instance of the fetch class. + The instance uses a space-separated list of URLs from the + SRC_URI + variable and then calls the download + method to download the files. + + + + The instantiation of the fetch class is usually followed by: + + rootdir = l.getVar('WORKDIR', True) + fetcher.unpack(rootdir) + + This code unpacks the downloaded files to the + specified by WORKDIR. + + For convenience, the naming in these examples matches + the variables used by OpenEmbedded. + If you want to see the above code in action, examine + the OpenEmbedded class file base.bbclass. + + The SRC_URI and WORKDIR + variables are not hardcoded into the fetcher, since those fetcher + methods can be (and are) called with different variable names. + In OpenEmbedded for example, the shared state (sstate) code uses + the fetch module to fetch the sstate files. + + + + When the download() method is called, + BitBake tries to resolve the URLs by looking for source files + in a specific search order: + + Pre-mirror Sites: + BitBake first uses pre-mirrors to try and find source files. + These locations are defined using the + PREMIRRORS + variable. + + Source URI: + If pre-mirrors fail, BitBake uses the original URL (e.g from + SRC_URI). + + Mirror Sites: + If fetch failures occur, BitBake next uses mirror locations as + defined by the + MIRRORS + variable. + + + + + + For each URL passed to the fetcher, the fetcher + calls the submodule that handles that particular URL type. + This behavior can be the source of some confusion when you + are providing URLs for the SRC_URI + variable. + Consider the following two URLs: + + http://git.yoctoproject.org/git/poky;protocol=git + git://git.yoctoproject.org/git/poky;protocol=http + + In the former case, the URL is passed to the + wget fetcher, which does not + understand "git". + Therefore, the latter case is the correct form since the + Git fetcher does know how to use HTTP as a transport. + + + + Here are some examples that show commonly used mirror + definitions: + + PREMIRRORS ?= "\ + bzr://.*/.* http://somemirror.org/sources/ \n \ + cvs://.*/.* http://somemirror.org/sources/ \n \ + git://.*/.* http://somemirror.org/sources/ \n \ + hg://.*/.* http://somemirror.org/sources/ \n \ + osc://.*/.* http://somemirror.org/sources/ \n \ + p4://.*/.* http://somemirror.org/sources/ \n \ + svn://.*/.* http://somemirror.org/sources/ \n" + + MIRRORS =+ "\ + ftp://.*/.* http://somemirror.org/sources/ \n \ + http://.*/.* http://somemirror.org/sources/ \n \ + https://.*/.* http://somemirror.org/sources/ \n" + + It is useful to note that BitBake supports + cross-URLs. + It is possible to mirror a Git repository on an HTTP + server as a tarball. + This is what the git:// mapping in + the previous example does. + + + + Since network accesses are slow, Bitbake maintains a + cache of files downloaded from the network. + Any source files that are not local (i.e. + downloaded from the Internet) are placed into the download + directory, which is specified by the + DL_DIR + variable. + + + + File integrity is of key importance for reproducing builds. + For non-local archive downloads, the fetcher code can verify + SHA-256 and MD5 checksums to ensure the archives have been + downloaded correctly. + You can specify these checksums by using the + SRC_URI variable with the appropriate + varflags as follows: + + SRC_URI[md5sum] = "value" + SRC_URI[sha256sum] = "value" + + You can also specify the checksums as parameters on the + SRC_URI as shown below: + + SRC_URI = "http://example.com/foobar.tar.bz2;md5sum=4a8e0f237e961fd7785d19d07fdb994d" + + If multiple URIs exist, you can specify the checksums either + directly as in the previous example, or you can name the URLs. + The following syntax shows how you name the URIs: + + SRC_URI = "http://example.com/foobar.tar.bz2;name=foo" + SRC_URI[foo.md5sum] = 4a8e0f237e961fd7785d19d07fdb994d + + After a file has been downloaded and has had its checksum checked, + a ".done" stamp is placed in DL_DIR. + BitBake uses this stamp during subsequent builds to avoid + downloading or comparing a checksum for the file again. + + It is assumed that local storage is safe from data corruption. + If this were not the case, there would be bigger issues to worry about. + + + + + If + BB_STRICT_CHECKSUM + is set, any download without a checksum triggers an + error message. + The + BB_NO_NETWORK + variable can be used to make any attempted network access a fatal + error, which is useful for checking that mirrors are complete + as well as other things. + +
+ +
+ The Unpack + + + The unpack process usually immediately follows the download. + For all URLs except Git URLs, BitBake uses the common + unpack method. + + + + A number of parameters exist that you can specify within the + URL to govern the behavior of the unpack stage: + + unpack: + Controls whether the URL components are unpacked. + If set to "1", which is the default, the components + are unpacked. + If set to "0", the unpack stage leaves the file alone. + This parameter is useful when you want an archive to be + copied in and not be unpacked. + + dos: + Applies to .zip and + .jar files and specifies whether to + use DOS line ending conversion on text files. + + basepath: + Instructs the unpack stage to strip the specified + directories from the source path when unpacking. + + subdir: + Unpacks the specific URL to the specified subdirectory + within the root directory. + + + The unpack call automatically decompresses and extracts files + with ".Z", ".z", ".gz", ".xz", ".zip", ".jar", ".ipk", ".rpm". + ".srpm", ".deb" and ".bz2" extensions as well as various combinations + of tarball extensions. + + + + As mentioned, the Git fetcher has its own unpack method that + is optimized to work with Git trees. + Basically, this method works by cloning the tree into the final + directory. + The process is completed using references so that there is + only one central copy of the Git metadata needed. + +
+ +
+ Fetchers + + + As mentioned earlier, the URL prefix determines which + fetcher submodule BitBake uses. + Each submodule can support different URL parameters, + which are described in the following sections. + + +
+ Local file fetcher (<filename>file://</filename>) + + + This submodule handles URLs that begin with + file://. + The filename you specify within the URL can be + either an absolute or relative path to a file. + If the filename is relative, the contents of the + FILESPATH + variable is used in the same way + PATH is used to find executables. + Failing that, + FILESDIR + is used to find the appropriate relative file. + + FILESDIR is deprecated and can + be replaced with FILESPATH. + Because FILESDIR is likely to be + removed, you should not use this variable in any new code. + + If the file cannot be found, it is assumed that it is available in + DL_DIR + by the time the download() method is called. + + + + If you specify a directory, the entire directory is + unpacked. + + + + Here are a couple of example URLs, the first relative and + the second absolute: + + SRC_URI = "file://relativefile.patch" + SRC_URI = "file:///Users/ich/very_important_software" + + +
+ +
+ HTTP/FTP wget fetcher (<filename>http://</filename>, <filename>ftp://</filename>, <filename>https://</filename>) + + + This fetcher obtains files from web and FTP servers. + Internally, the fetcher uses the wget utility. + + + + The executable and parameters used are specified by the + FETCHCMD_wget variable, which defaults + to sensible values. + The fetcher supports a parameter "downloadfilename" that + allows the name of the downloaded file to be specified. + Specifying the name of the downloaded file is useful + for avoiding collisions in + DL_DIR + when dealing with multiple files that have the same name. + + + + Some example URLs are as follows: + + SRC_URI = "http://oe.handhelds.org/not_there.aac" + SRC_URI = "ftp://oe.handhelds.org/not_there_as_well.aac" + SRC_URI = "ftp://you@oe.handhelds.org/home/you/secret.plan" + + + + Because URL parameters are delimited by semi-colons, this can + introduce ambiguity when parsing URLs that also contain semi-colons, + for example: + + SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git;a=snapshot;h=a5dd47" + + Such URLs should should be modified by replacing semi-colons with '&' characters: + + SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47" + + In most cases this should work. Treating semi-colons and '&' in queries + identically is recommended by the World Wide Web Consortium (W3C). + Note that due to the nature of the URL, you may have to specify the name + of the downloaded file as well: + + SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47;downloadfilename=myfile.bz2" + + +
+ +
+ CVS fetcher (<filename>(cvs://</filename>) + + + This submodule handles checking out files from the + CVS version control system. + You can configure it using a number of different variables: + + FETCHCMD_cvs: + The name of the executable to use when running + the cvs command. + This name is usually "cvs". + + SRCDATE: + The date to use when fetching the CVS source code. + A special value of "now" causes the checkout to + be updated on every build. + + CVSDIR: + Specifies where a temporary checkout is saved. + The location is often DL_DIR/cvs. + + CVS_PROXY_HOST: + The name to use as a "proxy=" parameter to the + cvs command. + + CVS_PROXY_PORT: + The port number to use as a "proxyport=" parameter to + the cvs command. + + + As well as the standard username and password URL syntax, + you can also configure the fetcher with various URL parameters: + + + + The supported parameters are as follows: + + "method": + The protocol over which to communicate with the CVS server. + By default, this protocol is "pserver". + If "method" is set to "ext", BitBake examines the + "rsh" parameter and sets CVS_RSH. + You can use "dir" for local directories. + + "module": + Specifies the module to check out. + You must supply this parameter. + + "tag": + Describes which CVS TAG should be used for + the checkout. + By default, the TAG is empty. + + "date": + Specifies a date. + If no "date" is specified, the + SRCDATE + of the configuration is used to checkout a specific date. + The special value of "now" causes the checkout to be + updated on every build. + + "localdir": + Used to rename the module. + Effectively, you are renaming the output directory + to which the module is unpacked. + You are forcing the module into a special + directory relative to + CVSDIR. + + "rsh" + Used in conjunction with the "method" parameter. + + "scmdata": + Causes the CVS metadata to be maintained in the tarball + the fetcher creates when set to "keep". + The tarball is expanded into the work directory. + By default, the CVS metadata is removed. + + "fullpath": + Controls whether the resulting checkout is at the + module level, which is the default, or is at deeper + paths. + + "norecurse": + Causes the fetcher to only checkout the specified + directory with no recurse into any subdirectories. + + "port": + The port to which the CVS server connects. + + + Some example URLs are as follows: + + SRC_URI = "cvs://CVSROOT;module=mymodule;tag=some-version;method=ext" + SRC_URI = "cvs://CVSROOT;module=mymodule;date=20060126;localdir=usethat" + + +
+ +
+ Subversion (SVN) Fetcher (<filename>svn://</filename>) + + + This fetcher submodule fetches code from the + Subversion source control system. + The executable used is specified by + FETCHCMD_svn, which defaults + to "svn". + The fetcher's temporary working directory is set by + SVNDIR, + which is usually DL_DIR/svn. + + + + The supported parameters are as follows: + + "module": + The name of the svn module to checkout. + You must provide this parameter. + You can think of this parameter as the top-level + directory of the repository data you want. + + "protocol": + The protocol to use, which defaults to "svn". + Other options are "svn+ssh" and "rsh". + For "rsh", the "rsh" parameter is also used. + + "rev": + The revision of the source code to checkout. + + "date": + The date of the source code to checkout. + Specific revisions are generally much safer to checkout + rather than by date as they do not involve timezones + (e.g. they are much more deterministic). + + "scmdata": + Causes the “.svn†directories to be available during + compile-time when set to "keep". + By default, these directories are removed. + + "transportuser": + When required, sets the username for the transport. + By default, this parameter is empty. + The transport username is different than the username + used in the main URL, which is passed to the subversion + command. + + + Following are two examples using svn: + + SRC_URI = "svn://svn.oe.handhelds.org/svn;module=vip;proto=http;rev=667" + SRC_URI = "svn://svn.oe.handhelds.org/svn/;module=opie;proto=svn+ssh;date=20060126" + + +
+ +
+ Git Fetcher (<filename>git://</filename>) + + + This fetcher submodule fetches code from the Git + source control system. + The fetcher works by creating a bare clone of the + remote into + GITDIR, + which is usually DL_DIR/git2. + This bare clone is then cloned into the work directory during the + unpack stage when a specific tree is checked out. + This is done using alternates and by reference to + minimize the amount of duplicate data on the disk and + make the unpack process fast. + The executable used can be set with + FETCHCMD_git. + + + + This fetcher supports the following parameters: + + "protocol": + The protocol used to fetch the files. + The default is "git" when a hostname is set. + If a hostname is not set, the Git protocol is "file". + You can also use "http", "https", "ssh" and "rsync". + + "nocheckout": + Tells the fetcher to not checkout source code when + unpacking when set to "1". + Set this option for the URL where there is a custom + routine to checkout code. + The default is "0". + + "rebaseable": + Indicates that the upstream Git repository can be rebased. + You should set this parameter to "1" if + revisions can become detached from branches. + In this case, the source mirror tarball is done per + revision, which has a loss of efficiency. + Rebasing the upstream Git repository could cause the + current revision to disappear from the upstream repository. + This option reminds the fetcher to preserve the local cache + carefully for future use. + The default value for this parameter is "0". + + "nobranch": + Tells the fetcher to not check the SHA validation + for the branch when set to "1". + The default is "0". + Set this option for the recipe that refers to + the commit that is valid for a tag instead of + the branch. + + "bareclone": + Tells the fetcher to clone a bare clone into the + destination directory without checking out a working tree. + Only the raw Git metadata is provided. + This parameter implies the "nocheckout" parameter as well. + + "branch": + The branch(es) of the Git tree to clone. + If unset, this is assumed to be "master". + The number of branch parameters much match the number of + name parameters. + + "rev": + The revision to use for the checkout. + The default is "master". + + "tag": + Specifies a tag to use for the checkout. + To correctly resolve tags, BitBake must access the + network. + For that reason, tags are often not used. + As far as Git is concerned, the "tag" parameter behaves + effectively the same as the "rev" parameter. + + "subpath": + Limits the checkout to a specific subpath of the tree. + By default, the whole tree is checked out. + + "destsuffix": + The name of the path in which to place the checkout. + By default, the path is git/. + + + Here are some example URLs: + + SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1" + SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http" + + +
+ +
+ Git Submodule Fetcher (<filename>gitsm://</filename>) + + + This fetcher submodule inherits from the + Git fetcher and extends + that fetcher's behavior by fetching a repository's submodules. + SRC_URI + is passed to the Git fetcher as described in the + "Git Fetcher (git://)" + section. + + Notes and Warnings + + You must clean a recipe when switching between + 'git://' and + 'gitsm://' URLs. + + + + The Git Submodules fetcher is not a complete fetcher + implementation. + The fetcher has known issues where it does not use the + normal source mirroring infrastructure properly. + + + +
+ +
+ ClearCase Fetcher (<filename>ccrc://</filename>) + + + This fetcher submodule fetches code from a + ClearCase + repository. + + + + To use this fetcher, make sure your recipe has proper + SRC_URI, + SRCREV, and + PV settings. + Here is an example: + + SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module" + SRCREV = "EXAMPLE_CLEARCASE_TAG" + PV = "${@d.getVar("SRCREV", False).replace("/", "+")}" + + The fetcher uses the rcleartool or + cleartool remote client, depending on + which one is available. + + + + Following are options for the SRC_URI + statement: + + vob: + The name, which must include the + prepending "/" character, of the ClearCase VOB. + This option is required. + + module: + The module, which must include the + prepending "/" character, in the selected VOB. + + The module and vob + options are combined to create the load rule in + the view config spec. + As an example, consider the vob and + module values from the + SRC_URI statement at the start of this section. + Combining those values results in the following: + + load /example_vob/example_module + + + + proto: + The protocol, which can be either http or + https. + + + + + + By default, the fetcher creates a configuration specification. + If you want this specification written to an area other than the default, + use the CCASE_CUSTOM_CONFIG_SPEC variable + in your recipe to define where the specification is written. + + the SRCREV loses its functionality if you + specify this variable. + However, SRCREV is still used to label the + archive after a fetch even though it does not define what is + fetched. + + + + + Here are a couple of other behaviors worth mentioning: + + + When using cleartool, the login of + cleartool is handled by the system. + The login require no special steps. + + + In order to use rcleartool with authenticated + users, an "rcleartool login" is necessary before using the fetcher. + + + +
+ +
+ Other Fetchers + + + Fetch submodules also exist for the following: + + + Bazaar (bzr://) + + + Perforce (p4://) + + + Trees using Git Annex (gitannex://) + + + Secure FTP (sftp://) + + + Secure Shell (ssh://) + + + Repo (repo://) + + + OSC (osc://) + + + Mercurial (hg://) + + + No documentation currently exists for these lesser used + fetcher submodules. + However, you might find the code helpful and readable. + +
+
+ +
+ Auto Revisions + + + We need to document AUTOREV and + SRCREV_FORMAT here. + +
+ diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml new file mode 100644 index 000000000..f6d82b4f3 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml @@ -0,0 +1,505 @@ + + + + Hello World Example + +
+ BitBake Hello World + + + The simplest example commonly used to demonstrate any new + programming language or tool is the + "Hello World" + example. + This appendix demonstrates, in tutorial form, Hello + World within the context of BitBake. + The tutorial describes how to create a new project + and the applicable metadata files necessary to allow + BitBake to build it. + +
+ +
+ Obtaining BitBake + + + See the + "Obtaining BitBake" + section for information on how to obtain BitBake. + Once you have the source code on your machine, the BitBake directory + appears as follows: + + $ ls -al + total 100 + drwxrwxr-x. 9 wmat wmat 4096 Jan 31 13:44 . + drwxrwxr-x. 3 wmat wmat 4096 Feb 4 10:45 .. + -rw-rw-r--. 1 wmat wmat 365 Nov 26 04:55 AUTHORS + drwxrwxr-x. 2 wmat wmat 4096 Nov 26 04:55 bin + drwxrwxr-x. 4 wmat wmat 4096 Jan 31 13:44 build + -rw-rw-r--. 1 wmat wmat 16501 Nov 26 04:55 ChangeLog + drwxrwxr-x. 2 wmat wmat 4096 Nov 26 04:55 classes + drwxrwxr-x. 2 wmat wmat 4096 Nov 26 04:55 conf + drwxrwxr-x. 3 wmat wmat 4096 Nov 26 04:55 contrib + -rw-rw-r--. 1 wmat wmat 17987 Nov 26 04:55 COPYING + drwxrwxr-x. 3 wmat wmat 4096 Nov 26 04:55 doc + -rw-rw-r--. 1 wmat wmat 69 Nov 26 04:55 .gitignore + -rw-rw-r--. 1 wmat wmat 849 Nov 26 04:55 HEADER + drwxrwxr-x. 5 wmat wmat 4096 Jan 31 13:44 lib + -rw-rw-r--. 1 wmat wmat 195 Nov 26 04:55 MANIFEST.in + -rw-rw-r--. 1 wmat wmat 2887 Nov 26 04:55 TODO + + + + + At this point, you should have BitBake cloned to + a directory that matches the previous listing except for + dates and user names. + +
+ +
+ Setting Up the BitBake Environment + + + First, you need to be sure that you can run BitBake. + Set your working directory to where your local BitBake + files are and run the following command: + + $ ./bin/bitbake --version + BitBake Build Tool Core version 1.23.0, bitbake version 1.23.0 + + The console output tells you what version you are running. + + + + The recommended method to run BitBake is from a directory of your + choice. + To be able to run BitBake from any directory, you need to add the + executable binary to your binary to your shell's environment + PATH variable. + First, look at your current PATH variable + by entering the following: + + $ echo $PATH + + Next, add the directory location for the BitBake binary to the + PATH. + Here is an example that adds the + /home/scott-lenovo/bitbake/bin directory + to the front of the PATH variable: + + $ export PATH=/home/scott-lenovo/bitbake/bin:$PATH + + You should now be able to enter the bitbake + command from the command line while working from any directory. + +
+ +
+ The Hello World Example + + + The overall goal of this exercise is to build a + complete "Hello World" example utilizing task and layer + concepts. + Because this is how modern projects such as OpenEmbedded and + the Yocto Project utilize BitBake, the example + provides an excellent starting point for understanding + BitBake. + + + + To help you understand how to use BitBake to build targets, + the example starts with nothing but the bitbake + command, which causes BitBake to fail and report problems. + The example progresses by adding pieces to the build to + eventually conclude with a working, minimal "Hello World" + example. + + + + While every attempt is made to explain what is happening during + the example, the descriptions cannot cover everything. + You can find further information throughout this manual. + Also, you can actively participate in the + + discussion mailing list about the BitBake build tool. + + + + This example was inspired by and drew heavily from these sources: + + + Mailing List post - The BitBake equivalent of "Hello, World!" + + + Hambedded Linux blog post - From Bitbake Hello World to an Image + + + + + + As stated earlier, the goal of this example + is to eventually compile "Hello World". + However, it is unknown what BitBake needs and what you have + to provide in order to achieve that goal. + Recall that BitBake utilizes three types of metadata files: + Configuration Files, + Classes, and + Recipes. + But where do they go? + How does BitBake find them? + BitBake's error messaging helps you answer these types of questions + and helps you better understand exactly what is going on. + + + + Following is the complete "Hello World" example. + + + + Create a Project Directory: + First, set up a directory for the "Hello World" project. + Here is how you can do so in your home directory: + + $ mkdir ~/hello + $ cd ~/hello + + This is the directory that BitBake will use to do all of + its work. + You can use this directory to keep all the metafiles needed + by BitBake. + Having a project directory is a good way to isolate your + project. + + Run Bitbake: + At this point, you have nothing but a project directory. + Run the bitbake command and see what + it does: + + $ bitbake + The BBPATH variable is not set and bitbake did not + find a conf/bblayers.conf file in the expected location. + Maybe you accidentally invoked bitbake from the wrong directory? + DEBUG: Removed the following variables from the environment: + GNOME_DESKTOP_SESSION_ID, XDG_CURRENT_DESKTOP, + GNOME_KEYRING_CONTROL, DISPLAY, SSH_AGENT_PID, LANG, no_proxy, + XDG_SESSION_PATH, XAUTHORITY, SESSION_MANAGER, SHLVL, + MANDATORY_PATH, COMPIZ_CONFIG_PROFILE, WINDOWID, EDITOR, + GPG_AGENT_INFO, SSH_AUTH_SOCK, GDMSESSION, GNOME_KEYRING_PID, + XDG_SEAT_PATH, XDG_CONFIG_DIRS, LESSOPEN, DBUS_SESSION_BUS_ADDRESS, + _, XDG_SESSION_COOKIE, DESKTOP_SESSION, LESSCLOSE, DEFAULTS_PATH, + UBUNTU_MENUPROXY, OLDPWD, XDG_DATA_DIRS, COLORTERM, LS_COLORS + + The majority of this output is specific to environment variables + that are not directly relevant to BitBake. + However, the very first message regarding the + BBPATH variable and the + conf/bblayers.conf file + is relevant. + + When you run BitBake, it begins looking for metadata files. + The + BBPATH + variable is what tells BitBake where to look for those files. + BBPATH is not set and you need to set it. + Without BBPATH, Bitbake cannot + find any configuration files (.conf) + or recipe files (.bb) at all. + BitBake also cannot find the bitbake.conf + file. + + Setting BBPATH: + For this example, you can set BBPATH + in the same manner that you set PATH + earlier in the appendix. + You should realize, though, that it is much more flexible to set the + BBPATH variable up in a configuration + file for each project. + From your shell, enter the following commands to set and + export the BBPATH variable: + + $ BBPATH="projectdirectory" + $ export BBPATH + + Use your actual project directory in the command. + BitBake uses that directory to find the metadata it needs for + your project. + + When specifying your project directory, do not use the + tilde ("~") character as BitBake does not expand that character + as the shell would. + + + Run Bitbake: + Now that you have BBPATH defined, run + the bitbake command again: + + $ bitbake + ERROR: Traceback (most recent call last): + File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped + return func(fn, *args) + File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 173, in parse_config_file + return bb.parse.handle(fn, data, include) + File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 99, in handle + return h['handle'](fn, data, include) + File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 120, in handle + abs_fn = resolve_file(fn, data) + File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 117, in resolve_file + raise IOError("file %s not found in %s" % (fn, bbpath)) + IOError: file conf/bitbake.conf not found in /home/scott-lenovo/hello + + ERROR: Unable to parse conf/bitbake.conf: file conf/bitbake.conf not found in /home/scott-lenovo/hello + + This sample output shows that BitBake could not find the + conf/bitbake.conf file in the project + directory. + This file is the first thing BitBake must find in order + to build a target. + And, since the project directory for this example is + empty, you need to provide a conf/bitbake.conf + file. + + Creating conf/bitbake.conf: + The conf/bitbake.conf includes a number of + configuration variables BitBake uses for metadata and recipe + files. + For this example, you need to create the file in your project directory + and define some key BitBake variables. + For more information on the bitbake.conf, + see + + + Use the following commands to create the conf + directory in the project directory: + + $ mkdir conf + + From within the conf directory, use + some editor to create the bitbake.conf + so that it contains the following: + + TMPDIR = "${TOPDIR}/tmp" + CACHE = "${TMPDIR}/cache" + STAMP = "${TMPDIR}/stamps" + T = "${TMPDIR}/work" + B = "${TMPDIR}" + + The TMPDIR variable establishes a directory + that BitBake uses for build output and intermediate files (other + than the cached information used by the + Setscene process. + Here, the TMPDIR directory is set to + hello/tmp. + Tip + You can always safely delete the tmp + directory in order to rebuild a BitBake target. + The build process creates the directory for you + when you run BitBake. + + For information about each of the other variables defined in this + example, click on the links to take you to the definitions in + the glossary. + + Run Bitbake: + After making sure that the conf/bitbake.conf + file exists, you can run the bitbake + command again: + +$ bitbake +ERROR: Traceback (most recent call last): + File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped + return func(fn, *args) + File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit + bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data) + File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit + include(fn, file, lineno, d, "inherit") + File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include + raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno) +ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass + +ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass + + In the sample output, BitBake could not find the + classes/base.bbclass file. + You need to create that file next. + + Creating classes/base.bbclass: + BitBake uses class files to provide common code and functionality. + The minimally required class for BitBake is the + classes/base.bbclass file. + The base class is implicitly inherited by + every recipe. + BitBake looks for the class in the classes + directory of the project (i.e hello/classes + in this example). + + Create the classes directory as follows: + + $ cd $HOME/hello + $ mkdir classes + + Move to the classes directory and then + create the base.bbclass file by inserting + this single line: + + addtask build + + The minimal task that BitBake runs is the + do_build task. + This is all the example needs in order to build the project. + Of course, the base.bbclass can have much + more depending on which build environments BitBake is + supporting. + For more information on the base.bbclass file, + you can look at + . + + Run Bitbake: + After making sure that the classes/base.bbclass + file exists, you can run the bitbake + command again: + + $ bitbake + Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information. + + BitBake is finally reporting no errors. + However, you can see that it really does not have anything + to do. + You need to create a recipe that gives BitBake something to do. + + Creating a Layer: + While it is not really necessary for such a small example, + it is good practice to create a layer in which to keep your + code separate from the general metadata used by BitBake. + Thus, this example creates and uses a layer called "mylayer". + + You can find additional information on adding a layer at + . + + + Minimally, you need a recipe file and a layer configuration + file in your layer. + The configuration file needs to be in the conf + directory inside the layer. + Use these commands to set up the layer and the conf + directory: + + $ cd $HOME + $ mkdir mylayer + $ cd mylayer + $ mkdir conf + + Move to the conf directory and create a + layer.conf file that has the following: + + BBPATH .= ":${LAYERDIR}" + + BBFILES += "${LAYERDIR}/*.bb" + + BBFILE_COLLECTIONS += "mylayer" + BBFILE_PATTERN_mylayer := "^${LAYERDIR}/" + + For information on these variables, click the links + to go to the definitions in the glossary. + You need to create the recipe file next. + Inside your layer at the top-level, use an editor and create + a recipe file named printhello.bb that + has the following: + + DESCRIPTION = "Prints Hello World" + PN = 'printhello' + PV = '1' + + python do_build() { + bb.plain("********************"); + bb.plain("* *"); + bb.plain("* Hello, World! *"); + bb.plain("* *"); + bb.plain("********************"); + } + + The recipe file simply provides a description of the + recipe, the name, version, and the do_build + task, which prints out "Hello World" to the console. + For more information on these variables, follow the links + to the glossary. + + Run Bitbake With a Target: + Now that a BitBake target exists, run the command and provide + that target: + + $ cd $HOME/hello + $ bitbake printhello + ERROR: no recipe files to build, check your BBPATH and BBFILES? + + Summary: There was 1 ERROR message shown, returning a non-zero exit code. + + We have created the layer with the recipe and the layer + configuration file but it still seems that BitBake cannot + find the recipe. + BitBake needs a conf/bblayers.conf that + lists the layers for the project. + Without this file, BitBake cannot find the recipe. + + Creating conf/bblayers.conf: + BitBake uses the conf/bblayers.conf file + to locate layers needed for the project. + This file must reside in the conf directory + of the project (i.e. hello/conf for this + example). + Set your working directory to the hello/conf + directory and then create the bblayers.conf + file so that it contains the following: + + BBLAYERS ?= " \ + /home/<you>/mylayer \ + " + + You need to provide your own information for + you in the file. + + Run Bitbake With a Target: + Now that you have supplied the bblayers.conf + file, run the bitbake command and provide + the target: + + $ bitbake printhello + Parsing recipes: 100% |##################################################################################| + Time: 00:00:00 + Parsing of 1 .bb files complete (0 cached, 1 parsed). 1 targets, 0 skipped, 0 masked, 0 errors. + NOTE: Resolving any missing task queue dependencies + NOTE: Preparing RunQueue + NOTE: Executing RunQueue Tasks + ******************** + * * + * Hello, World! * + * * + ******************** + NOTE: Tasks Summary: Attempted 1 tasks of which 0 didn't need to be rerun and all succeeded. + + BitBake finds the printhello recipe and + successfully runs the task. + + After the first execution, re-running + bitbake printhello again will not + result in a BitBake run that prints the same console + output. + The reason for this is that the first time the + printhello.bb recipe's + do_build task executes + successfully, BitBake writes a stamp file for the task. + Thus, the next time you attempt to run the task + using that same bitbake command, + BitBake notices the stamp and therefore determines + that the task does not need to be re-run. + If you delete the tmp directory + or run bitbake -c clean printhello + and then re-run the build, the "Hello, World!" message will + be printed again. + + + +
+ diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml new file mode 100644 index 000000000..7a37edd50 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml @@ -0,0 +1,691 @@ + + + + Overview + + + Welcome to the BitBake User Manual. + This manual provides information on the BitBake tool. + The information attempts to be as independent as possible regarding + systems that use BitBake, such as OpenEmbedded and the + Yocto Project. + In some cases, scenarios or examples within the context of + a build system are used in the manual to help with understanding. + For these cases, the manual clearly states the context. + + +
+ Introduction + + + Fundamentally, BitBake is a generic task execution + engine that allows shell and Python tasks to be run + efficiently and in parallel while working within + complex inter-task dependency constraints. + One of BitBake's main users, OpenEmbedded, takes this core + and builds embedded Linux software stacks using + a task-oriented approach. + + + + Conceptually, BitBake is similar to GNU Make in + some regards but has significant differences: + + + BitBake executes tasks according to provided + metadata that builds up the tasks. + Metadata is stored in recipe (.bb) + and related recipe "append" (.bbappend) + files, configuration (.conf) and + underlying include (.inc) files, and + in class (.bbclass) files. + The metadata provides + BitBake with instructions on what tasks to run and + the dependencies between those tasks. + + + BitBake includes a fetcher library for obtaining source + code from various places such as local files, source control + systems, or websites. + + + The instructions for each unit to be built (e.g. a piece + of software) are known as "recipe" files and + contain all the information about the unit + (dependencies, source file locations, checksums, description + and so on). + + + BitBake includes a client/server abstraction and can + be used from a command line or used as a service over + XML-RPC and has several different user interfaces. + + + +
+ +
+ History and Goals + + + BitBake was originally a part of the OpenEmbedded project. + It was inspired by the Portage package management system + used by the Gentoo Linux distribution. + On December 7, 2004, OpenEmbedded project team member + Chris Larson split the project into two distinct pieces: + + BitBake, a generic task executor + OpenEmbedded, a metadata set utilized by + BitBake + + Today, BitBake is the primary basis of the + OpenEmbedded + project, which is being used to build and maintain Linux + distributions such as the + Angstrom Distribution, + and which is also being used as the build tool for Linux projects + such as the + Yocto Project. + + + + Prior to BitBake, no other build tool adequately met the needs of + an aspiring embedded Linux distribution. + All of the build systems used by traditional desktop Linux + distributions lacked important functionality, and none of the + ad hoc Buildroot-based systems, prevalent in the + embedded space, were scalable or maintainable. + + + + Some important original goals for BitBake were: + + + Handle cross-compilation. + + + Handle inter-package dependencies (build time on + target architecture, build time on native + architecture, and runtime). + + + Support running any number of tasks within a given + package, including, but not limited to, fetching + upstream sources, unpacking them, patching them, + configuring them, and so forth. + + + Be Linux distribution agnostic for both build and + target systems. + + + Be architecture agnostic. + + + Support multiple build and target operating systems + (e.g. Cygwin, the BSDs, and so forth). + + + Be self contained, rather than tightly + integrated into the build machine's root + filesystem. + + + Handle conditional metadata on the target architecture, + operating system, distribution, and machine. + + + Be easy to use the tools to supply local metadata and packages + against which to operate. + + + Be easy to use BitBake to collaborate between multiple + projects for their builds. + + + Provide an inheritance mechanism to share + common metadata between many packages. + + + Over time it became apparent that some further requirements + were necessary: + + + Handle variants of a base recipe (e.g. native, sdk, + and multilib). + + + Split metadata into layers and allow layers + to enhance or override other layers. + + + Allow representation of a given set of input variables + to a task as a checksum. + Based on that checksum, allow acceleration of builds + with prebuilt components. + + + BitBake satisfies all the original requirements and many more + with extensions being made to the basic functionality to + reflect the additional requirements. + Flexibility and power have always been the priorities. + BitBake is highly extensible and supports embedded Python code and + execution of any arbitrary tasks. + +
+ +
+ Concepts + + + BitBake is a program written in the Python language. + At the highest level, BitBake interprets metadata, decides + what tasks are required to run, and executes those tasks. + Similar to GNU Make, BitBake controls how software is + built. + GNU Make achieves its control through "makefiles", while + BitBake uses "recipes". + + + + BitBake extends the capabilities of a simple + tool like GNU Make by allowing for the definition of much more + complex tasks, such as assembling entire embedded Linux + distributions. + + + + The remainder of this section introduces several concepts + that should be understood in order to better leverage + the power of BitBake. + + +
+ Recipes + + + BitBake Recipes, which are denoted by the file extension + .bb, are the most basic metadata files. + These recipe files provide BitBake with the following: + + Descriptive information about the + package (author, homepage, license, and so on) + The version of the recipe + Existing dependencies (both build + and runtime dependencies) + Where the source code resides and + how to fetch it + Whether the source code requires + any patches, where to find them, and how to apply + them + How to configure and compile the + source code + Where on the target machine to install the + package or packages created + + + + + Within the context of BitBake, or any project utilizing BitBake + as its build system, files with the .bb + extension are referred to as recipes. + + The term "package" is also commonly used to describe recipes. + However, since the same word is used to describe packaged + output from a project, it is best to maintain a single + descriptive term - "recipes". + Put another way, a single "recipe" file is quite capable + of generating a number of related but separately installable + "packages". + In fact, that ability is fairly common. + + +
+ +
+ Configuration Files + + + Configuration files, which are denoted by the + .conf extension, define + various configuration variables that govern the project's build + process. + These files fall into several areas that define + machine configuration options, distribution configuration + options, compiler tuning options, general common + configuration options, and user configuration options. + The main configuration file is the sample + bitbake.conf file, which is + located within the BitBake source tree + conf directory. + +
+ +
+ Classes + + + Class files, which are denoted by the + .bbclass extension, contain + information that is useful to share between metadata files. + The BitBake source tree currently comes with one class metadata file + called base.bbclass. + You can find this file in the + classes directory. + The base.bbclass class files is special since it + is always included automatically for all recipes + and classes. + This class contains definitions for standard basic tasks such + as fetching, unpacking, configuring (empty by default), + compiling (runs any Makefile present), installing (empty by + default) and packaging (empty by default). + These tasks are often overridden or extended by other classes + added during the project development process. + +
+ +
+ Layers + + + Layers allow you to isolate different types of + customizations from each other. + While you might find it tempting to keep everything in one layer + when working on a single project, the more modular you organize + your metadata, the easier it is to cope with future changes. + + + + To illustrate how you can use layers to keep things modular, + consider customizations you might make to support a specific target machine. + These types of customizations typically reside in a special layer, + rather than a general layer, called a Board Support Package (BSP) + Layer. + Furthermore, the machine customizations should be isolated from + recipes and metadata that support a new GUI environment, for + example. + This situation gives you a couple of layers: one for the machine + configurations and one for the GUI environment. + It is important to understand, however, that the BSP layer can still + make machine-specific additions to recipes within + the GUI environment layer without polluting the GUI layer itself + with those machine-specific changes. + You can accomplish this through a recipe that is a BitBake append + (.bbappend) file. + +
+ +
+ Append Files + + + Append files, which are files that have the + .bbappend file extension, extend or + override information in an existing recipe file. + + + + BitBake expects every append file to have a corresponding recipe file. + Furthermore, the append file and corresponding recipe file + must use the same root filename. + The filenames can differ only in the file type suffix used + (e.g. formfactor_0.0.bb and + formfactor_0.0.bbappend). + + + + Information in append files extends or + overrides the information in the underlying, + similarly-named recipe files. + + + + When you name an append file, you can use the + wildcard character (%) to allow for matching recipe names. + For example, suppose you have an append file named + as follows: + + busybox_1.21.%.bbappend + + That append file would match any busybox_1.21.x.bb + version of the recipe. + So, the append file would match the following recipe names: + + busybox_1.21.1.bb + busybox_1.21.2.bb + busybox_1.21.3.bb + + If the busybox recipe was updated to + busybox_1.3.0.bb, the append name would not + match. + However, if you named the append file + busybox_1.%.bbappend, then you would have a match. + + + + In the most general case, you could name the append file something as + simple as busybox_%.bbappend to be entirely + version independent. + +
+
+ +
+ Obtaining BitBake + + + You can obtain BitBake several different ways: + + Cloning BitBake: + Using Git to clone the BitBake source code repository + is the recommended method for obtaining BitBake. + Cloning the repository makes it easy to get bug fixes + and have access to stable branches and the master + branch. + Once you have cloned BitBake, you should use + the latest stable + branch for development since the master branch is for + BitBake development and might contain less stable changes. + + You usually need a version of BitBake + that matches the metadata you are using. + The metadata is generally backwards compatible but + not forward compatible. + Here is an example that clones the BitBake repository: + + $ git clone git://git.openembedded.org/bitbake + + This command clones the BitBake Git repository into a + directory called bitbake. + Alternatively, you can + designate a directory after the + git clone command + if you want to call the new directory something + other than bitbake. + Here is an example that names the directory + bbdev: + + $ git clone git://git.openembedded.org/bitbake bbdev + + Installation using your Distribution + Package Management System: + This method is not + recommended because the BitBake version that is + provided by your distribution, in most cases, + is several + releases behind a snapshot of the BitBake repository. + + Taking a snapshot of BitBake: + Downloading a snapshot of BitBake from the + source code repository gives you access to a known + branch or release of BitBake. + + Cloning the Git repository, as described earlier, + is the preferred method for getting BitBake. + Cloning the repository makes it easier to update as + patches are added to the stable branches. + + The following example downloads a snapshot of + BitBake version 1.17.0: + + $ wget http://git.openembedded.org/bitbake/snapshot/bitbake-1.17.0.tar.gz + $ tar zxpvf bitbake-1.17.0.tar.gz + + After extraction of the tarball using the tar utility, + you have a directory entitled + bitbake-1.17.0. + + Using the BitBake that Comes With Your + Build Checkout: + A final possibility for getting a copy of BitBake is that it + already comes with your checkout of a larger Bitbake-based build + system, such as Poky or Yocto Project. + Rather than manually checking out individual layers and + gluing them together yourself, you can check + out an entire build system. + The checkout will already include a version of BitBake that + has been thoroughly tested for compatibility with the other + components. + For information on how to check out a particular BitBake-based + build system, consult that build system's supporting documentation. + + + +
+ +
+ The BitBake Command + + + The bitbake command is the primary interface + to the BitBake tool. + This section presents the BitBake command syntax and provides + several execution examples. + + +
+ Usage and syntax + + + Following is the usage and syntax for BitBake: + + $ bitbake -h + Usage: bitbake [options] [recipename/target recipe:do_task ...] + + Executes the specified task (default is 'build') for a given set of target recipes (.bb files). + It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which + will provide the layer, BBFILES and other configuration information. + + Options: + --version show program's version number and exit + -h, --help show this help message and exit + -b BUILDFILE, --buildfile=BUILDFILE + Execute tasks from a specific .bb recipe directly. + WARNING: Does not handle any dependencies from other + recipes. + -k, --continue Continue as much as possible after an error. While the + target that failed and anything depending on it cannot + be built, as much as possible will be built before + stopping. + -a, --tryaltconfigs Continue with builds by trying to use alternative + providers where possible. + -f, --force Force the specified targets/task to run (invalidating + any existing stamp file). + -c CMD, --cmd=CMD Specify the task to execute. The exact options + available depend on the metadata. Some examples might + be 'compile' or 'populate_sysroot' or 'listtasks' may + give a list of the tasks available. + -C INVALIDATE_STAMP, --clear-stamp=INVALIDATE_STAMP + Invalidate the stamp for the specified task such as + 'compile' and then run the default task for the + specified target(s). + -r PREFILE, --read=PREFILE + Read the specified file before bitbake.conf. + -R POSTFILE, --postread=POSTFILE + Read the specified file after bitbake.conf. + -v, --verbose Output more log message data to the terminal. + -D, --debug Increase the debug level. You can specify this more + than once. + -n, --dry-run Don't execute, just go through the motions. + -S SIGNATURE_HANDLER, --dump-signatures=SIGNATURE_HANDLER + Dump out the signature construction information, with + no task execution. The SIGNATURE_HANDLER parameter is + passed to the handler. Two common values are none and + printdiff but the handler may define more/less. none + means only dump the signature, printdiff means compare + the dumped signature with the cached one. + -p, --parse-only Quit after parsing the BB recipes. + -s, --show-versions Show current and preferred versions of all recipes. + -e, --environment Show the global or per-recipe environment complete + with information about where variables were + set/changed. + -g, --graphviz Save dependency tree information for the specified + targets in the dot syntax. + -I EXTRA_ASSUME_PROVIDED, --ignore-deps=EXTRA_ASSUME_PROVIDED + Assume these dependencies don't exist and are already + provided (equivalent to ASSUME_PROVIDED). Useful to + make dependency graphs more appealing + -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS + Show debug logging for the specified logging domains + -P, --profile Profile the command and save reports. + -u UI, --ui=UI The user interface to use (depexp, goggle, hob, knotty + or ncurses - default knotty). + -t SERVERTYPE, --servertype=SERVERTYPE + Choose which server type to use (process or xmlrpc - + default process). + --token=XMLRPCTOKEN Specify the connection token to be used when + connecting to a remote server. + --revisions-changed Set the exit code depending on whether upstream + floating revisions have changed or not. + --server-only Run bitbake without a UI, only starting a server + (cooker) process. + -B BIND, --bind=BIND The name/address for the bitbake server to bind to. + --no-setscene Do not run any setscene tasks. sstate will be ignored + and everything needed, built. + --remote-server=REMOTE_SERVER + Connect to the specified server. + -m, --kill-server Terminate the remote server. + --observe-only Connect to a server as an observing-only client. + --status-only Check the status of the remote bitbake server. + -w WRITEEVENTLOG, --write-log=WRITEEVENTLOG + Writes the event log of the build to a bitbake event + json file. Use '' (empty string) to assign the name + automatically. + + +
+ +
+ Examples + + + This section presents some examples showing how to use BitBake. + + +
+ Executing a Task Against a Single Recipe + + + Executing tasks for a single recipe file is relatively simple. + You specify the file in question, and BitBake parses + it and executes the specified task. + If you do not specify a task, BitBake executes the default + task, which is "buildâ€. + BitBake obeys inter-task dependencies when doing + so. + + + + The following command runs the build task, which is + the default task, on the foo_1.0.bb + recipe file: + + $ bitbake -b foo_1.0.bb + + The following command runs the clean task on the + foo.bb recipe file: + + $ bitbake -b foo.bb -c clean + + + The "-b" option explicitly does not handle recipe + dependencies. + Other than for debugging purposes, it is instead + recommended that you use the syntax presented in the + next section. + + +
+ +
+ Executing Tasks Against a Set of Recipe Files + + + There are a number of additional complexities introduced + when one wants to manage multiple .bb + files. + Clearly there needs to be a way to tell BitBake what + files are available and, of those, which you + want to execute. + There also needs to be a way for each recipe + to express its dependencies, both for build-time and + runtime. + There must be a way for you to express recipe preferences + when multiple recipes provide the same functionality, or when + there are multiple versions of a recipe. + + + + The bitbake command, when not using + "--buildfile" or "-b" only accepts a "PROVIDES". + You cannot provide anything else. + By default, a recipe file generally "PROVIDES" its + "packagename" as shown in the following example: + + $ bitbake foo + + This next example "PROVIDES" the package name and also uses + the "-c" option to tell BitBake to just execute the + do_clean task: + + $ bitbake -c clean foo + + +
+ +
+ Generating Dependency Graphs + + + BitBake is able to generate dependency graphs using + the dot syntax. + You can convert these graphs into images using the + dot tool from + Graphviz. + + + + When you generate a dependency graph, BitBake writes four files + to the current working directory: + + package-depends.dot: + Shows BitBake's knowledge of dependencies between + runtime targets. + + pn-depends.dot: + Shows dependencies between build-time targets + (i.e. recipes). + + task-depends.dot: + Shows dependencies between tasks. + + pn-buildlist: + Shows a simple list of targets that are to be built. + + + + + + To stop depending on common depends, use the "-I" depend + option and BitBake omits them from the graph. + Leaving this information out can produce more readable graphs. + This way, you can remove from the graph + DEPENDS from inherited classes + such as base.bbclass. + + + + Here are two examples that create dependency graphs. + The second example omits depends common in OpenEmbedded from + the graph: + + $ bitbake -g foo + + $ bitbake -g -I virtual/kernel -I eglibc foo + + +
+
+
+ diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml new file mode 100644 index 000000000..6329cd6e4 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml @@ -0,0 +1,1918 @@ + + + + Syntax and Operators + + + Bitbake files have their own syntax. + The syntax has similarities to several + other languages but also has some unique features. + This section describes the available syntax and operators + as well as provides examples. + + +
+ Basic Syntax + + + This section provides some basic syntax examples. + + +
+ Basic Variable Setting + + + The following example sets VARIABLE to + "value". + This assignment occurs immediately as the statement is parsed. + It is a "hard" assignment. + + VARIABLE = "value" + + As expected, if you include leading or trailing spaces as part of + an assignment, the spaces are retained: + + VARIABLE = " value" + VARIABLE = "value " + + Setting VARIABLE to "" sets it to an empty string, + while setting the variable to " " sets it to a blank space + (i.e. these are not the same values). + + VARIABLE = "" + VARIABLE = " " + + +
+ +
+ Variable Expansion + + + BitBake supports variables referencing one another's + contents using a syntax that is similar to shell scripting. + Following is an example that results in A + containing "aval" and B evaluating to + "preavalpost" based on that current value of + A. + + A = "aval" + B = "pre${A}post" + + You should realize that whenever B is + referenced, its evaluation will depend on the state of + A at that time. + Thus, later evaluations of B in the + previous example could result in different values + depending on the value of A. + +
+ +
+ Setting a default value (?=) + + + You can use the "?=" operator to achieve a "softer" assignment + for a variable. + This type of assignment allows you to define a variable if it + is undefined when the statement is parsed, but to leave the + value alone if the variable has a value. + Here is an example: + + A ?= "aval" + + If A is set at the time this statement is parsed, + the variable retains its value. + However, if A is not set, + the variable is set to "aval". + + This assignment is immediate. + Consequently, if multiple "?=" assignments + to a single variable exist, the first of those ends up getting + used. + + +
+ +
+ Setting a weak default value (??=) + + + It is possible to use a "weaker" assignment than in the + previous section by using the "??=" operator. + This assignment behaves identical to "?=" except that the + assignment is made at the end of the parsing process rather + than immediately. + Consequently, when multiple "??=" assignments exist, the last + one is used. + Also, any "=" or "?=" assignment will override the value set with + "??=". + Here is an example: + + A ??= "somevalue" + A ??= "someothervalue" + + If A is set before the above statements are parsed, + the variable retains its value. + If A is not set, + the variable is set to "someothervalue". + + + + Again, this assignment is a "lazy" or "weak" assignment + because it does not occur until the end + of the parsing process. + +
+ +
+ Immediate variable expansion (:=) + + + The ":=" operator results in a variable's + contents being expanded immediately, + rather than when the variable is actually used: + + T = "123" + A := "${B} ${A} test ${T}" + T = "456" + B = "${T} bval" + C = "cval" + C := "${C}append" + + In this example, A contains + "test 123" because ${B} and + ${A} at the time of parsing are undefined, + which leaves "test 123". + And, the variable C + contains "cvalappend" since ${C} immediately + expands to "cval". + +
+ +
+ Appending (+=) and prepending (=+) With Spaces + + + Appending and prepending values is common and can be accomplished + using the "+=" and "=+" operators. + These operators insert a space between the current + value and prepended or appended value. + + + + These operators take immediate effect during parsing. + Here are some examples: + + B = "bval" + B += "additionaldata" + C = "cval" + C =+ "test" + + The variable B contains + "bval additionaldata" and C + contains "test cval". + +
+ +
+ Appending (.=) and Prepending (=.) Without Spaces + + + If you want to append or prepend values without an + inserted space, use the ".=" and "=." operators. + + + + These operators take immediate effect during parsing. + Here are some examples: + + B = "bval" + B .= "additionaldata" + C = "cval" + C =. "test" + + The variable B contains + "bvaladditionaldata" and + C contains "testcval". + +
+ +
+ Appending and Prepending (Override Style Syntax) + + + You can also append and prepend a variable's value + using an override style syntax. + When you use this syntax, no spaces are inserted. + + + + These operators differ from the ":=", ".=", "=.", "+=", and "=+" + operators in that their effects are deferred + until after parsing completes rather than being immediately + applied. + Here are some examples: + + B = "bval" + B_append = " additional data" + C = "cval" + C_prepend = "additional data " + D = "dval" + D_append = "additional data" + + The variable B becomes + "bval additional data" and C becomes + "additional data cval". + The variable D becomes + "dvaladditional data". + + You must control all spacing when you use the + override syntax. + + +
+ +
+ Removal (Override Style Syntax) + + + You can remove values from lists using the removal + override style syntax. + Specifying a value for removal causes all occurrences of that + value to be removed from the variable. + + + + When you use this syntax, BitBake expects one or more strings. + Surrounding spaces are removed as well. + Here is an example: + + FOO = "123 456 789 123456 123 456 123 456" + FOO_remove = "123" + FOO_remove = "456" + FOO2 = "abc def ghi abcdef abc def abc def" + FOO2_remove = "abc def" + + The variable FOO becomes + "789 123456" and FOO2 becomes + "ghi abcdef". + +
+ +
+ Variable Flag Syntax + + + Variable flags are BitBake's implementation of variable properties + or attributes. + It is a way of tagging extra information onto a variable. + You can find more out about variable flags in general in the + "Variable Flags" + section. + + + + You can define, append, and prepend values to variable flags. + All the standard syntax operations previously mentioned work + for variable flags except for override style syntax + (i.e. _prepend, _append, + and _remove). + + + + Here are some examples showing how to set variable flags: + + FOO[a] = "abc" + FOO[b] = "123" + FOO[a] += "456" + + The variable FOO has two flags: + a and b. + The flags are immediately set to "abc" and "123", respectively. + The a flag becomes "abc 456". + + + + No need exists to pre-define variable flags. + You can simply start using them. + One extremely common application + is to attach some brief documentation to a BitBake variable as + follows: + + CACHE[doc] = "The directory holding the cache of the metadata." + + +
+ +
+ Inline Python Variable Expansion + + + You can use inline Python variable expansion to + set variables. + Here is an example: + + DATE = "${@time.strftime('%Y%m%d',time.gmtime())}" + + This example results in the DATE + variable being set to the current date. + + + + Probably the most common use of this feature is to extract + the value of variables from BitBake's internal data dictionary, + d. + The following lines select the values of a package name + and its version number, respectively: + + PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" + PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}" + + +
+ +
+ Providing Pathnames + + + When specifying pathnames for use with BitBake, + do not use the tilde ("~") character as a shortcut + for your home directory. + Doing so might cause BitBake to not recognize the + path since BitBake does not expand this character in + the same way a shell would. + + + + Instead, provide a fuller path as the following + example illustrates: + + BBLAYERS ?= " \ + /home/scott-lenovo/LayerA \ + " + + +
+
+ +
+ Conditional Syntax (Overrides) + + + BitBake uses + OVERRIDES + to control what variables are overridden after BitBake + parses recipes and configuration files. + This section describes how you can use + OVERRIDES as conditional metadata, + talks about key expansion in relationship to + OVERRIDES, and provides some examples + to help with understanding. + + +
+ Conditional Metadata + + + You can use OVERRIDES to conditionally select + a specific version of a variable and to conditionally + append or prepend the value of a variable. + + Overrides can only use lower-case characters. + Additionally, underscores are not permitted in override names + as they are used to separate overrides from each other and + from the variable name. + + + Selecting a Variable: + The OVERRIDES variable is + a colon-character-separated list that contains items + for which you want to satisfy conditions. + Thus, if you have a variable that is conditional on “armâ€, and “arm†+ is in OVERRIDES, then the “armâ€-specific + version of the variable is used rather than the non-conditional + version. + Here is an example: + + OVERRIDES = "architecture:os:machine" + TEST = "default" + TEST_os = "osspecific" + TEST_nooverride = "othercondvalue" + + In this example, the OVERRIDES + variable lists three overrides: + "architecture", "os", and "machine". + The variable TEST by itself has a default + value of "default". + You select the os-specific version of the TEST + variable by appending the "os" override to the variable + (i.e.TEST_os). + + + + To better understand this, consider a practical example + that assumes an OpenEmbedded metadata-based Linux + kernel recipe file. + The following lines from the recipe file first set + the kernel branch variable KBRANCH + to a default value, then conditionally override that + value based on the architecture of the build: + + KBRANCH = "standard/base" + KBRANCH_qemuarm = "standard/arm-versatile-926ejs" + KBRANCH_qemumips = "standard/mti-malta32" + KBRANCH_qemuppc = "standard/qemuppc" + KBRANCH_qemux86 = "standard/common-pc/base" + KBRANCH_qemux86-64 = "standard/common-pc-64/base" + KBRANCH_qemumips64 = "standard/mti-malta64" + + + Appending and Prepending: + BitBake also supports append and prepend operations to + variable values based on whether a specific item is + listed in OVERRIDES. + Here is an example: + + DEPENDS = "glibc ncurses" + OVERRIDES = "machine:local" + DEPENDS_append_machine = "libmad" + + In this example, DEPENDS becomes + "glibc ncurses libmad". + + + + Again, using an OpenEmbedded metadata-based + kernel recipe file as an example, the + following lines will conditionally append to the + KERNEL_FEATURES variable based + on the architecture: + + KERNEL_FEATURES_append = " ${KERNEL_EXTRA_FEATURES}" + KERNEL_FEATURES_append_qemux86=" cfg/sound.scc cfg/paravirt_kvm.scc" + KERNEL_FEATURES_append_qemux86-64=" cfg/sound.scc cfg/paravirt_kvm.scc" + + + + +
+ +
+ Key Expansion + + + Key expansion happens when the BitBake datastore is finalized + just before BitBake expands overrides. + To better understand this, consider the following example: + + A${B} = "X" + B = "2" + A2 = "Y" + + In this case, after all the parsing is complete, and + before any overrides are handled, BitBake expands + ${B} into "2". + This expansion causes A2, which was + set to "Y" before the expansion, to become "X". + +
+ +
+ Examples + + + Despite the previous explanations that show the different forms of + variable definitions, it can be hard to work + out exactly what happens when variable operators, conditional + overrides, and unconditional overrides are combined. + This section presents some common scenarios along + with explanations for variable interactions that + typically confuse users. + + + + There is often confusion concerning the order in which + overrides and various "append" operators take effect. + Recall that an append or prepend operation using "_append" + and "_prepend" does not result in an immediate assignment + as would "+=", ".=", "=+", or "=.". + Consider the following example: + + OVERRIDES = "foo" + A = "Z" + A_foo_append = "X" + + For this case, A is + unconditionally set to "Z" and "X" is + unconditionally and immediately appended to the variable + A_foo. + Because overrides have not been applied yet, + A_foo is set to "X" due to the append + and A simply equals "Z". + + + + Applying overrides, however, changes things. + Since "foo" is listed in OVERRIDES, + the conditional variable A is replaced + with the "foo" version, which is equal to "X". + So effectively, A_foo replaces A. + + + + This next example changes the order of the override and + the append: + + OVERRIDES = "foo" + A = "Z" + A_append_foo = "X" + + For this case, before overrides are handled, + A is set to "Z" and A_append_foo + is set to "X". + Once the override for "foo" is applied, however, + A gets appended with "X". + Consequently, A becomes "ZX". + Notice that spaces are not appended. + + + + This next example has the order of the appends and overrides reversed + back as in the first example: + + OVERRIDES = "foo" + A = "Y" + A_foo_append = "Z" + A_foo_append += "X" + + For this case, before any overrides are resolved, + A is set to "Y" using an immediate assignment. + After this immediate assignment, A_foo is set + to "Z", and then further appended with + "X" leaving the variable set to "Z X". + Finally, applying the override for "foo" results in the conditional + variable A becoming "Z X" (i.e. + A is replaced with A_foo). + + + + This final example mixes in some varying operators: + + A = "1" + A_append = "2" + A_append = "3" + A += "4" + A .= "5" + + For this case, the type of append operators are affecting the + order of assignments as BitBake passes through the code + multiple times. + Initially, A is set to "1 45" because + of the three statements that use immediate operators. + After these assignments are made, BitBake applies the + _append operations. + Those operations result in A becoming "1 4523". + +
+
+ +
+ Sharing Functionality + + + BitBake allows for metadata sharing through include files + (.inc) and class files + (.bbclass). + For example, suppose you have a piece of common functionality + such as a task definition that you want to share between + more than one recipe. + In this case, creating a .bbclass + file that contains the common functionality and then using + the inherit directive in your recipes to + inherit the class would be a common way to share the task. + + + + This section presents the mechanisms BitBake provides to + allow you to share functionality between recipes. + Specifically, the mechanisms include include, + inherit, INHERIT, and + require directives. + + +
+ Locating Include and Class Files + + + BitBake uses the + BBPATH + variable to locate needed include and class files. + Additionally, BitBake searches the current directory for + include and require + directives. + + The BBPATH variable is analogous to + the environment variable PATH. + + + + + In order for include and class files to be found by BitBake, + they need to be located in a "classes" subdirectory that can + be found in BBPATH. + +
+ +
+ <filename>inherit</filename> Directive + + + When writing a recipe or class file, you can use the + inherit directive to inherit the + functionality of a class (.bbclass). + BitBake only supports this directive when used within recipe + and class files (i.e. .bb and + .bbclass). + + + + The inherit directive is a rudimentary + means of specifying what classes of functionality your + recipes require. + For example, you can easily abstract out the tasks involved in + building a package that uses Autoconf and Automake and put + those tasks into a class file that can be used by your recipe. + + + + As an example, your recipes could use the following directive + to inherit an autotools.bbclass file. + The class file would contain common functionality for using + Autotools that could be shared across recipes: + + inherit autotools + + In this case, BitBake would search for the directory + classes/autotools.bbclass + in BBPATH. + + You can override any values and functions of the + inherited class within your recipe by doing so + after the "inherit" statement. + + + + + If necessary, it is possible to inherit a class + conditionally by using + a variable expression after the inherit + statement. + Here is an example: + + inherit ${VARNAME} + + If VARNAME is going to be set, it needs + to be set before the inherit statement + is parsed. + One way to achieve a conditional inherit in this case is to use + overrides: + + VARIABLE = "" + VARIABLE_someoverride = "myclass" + + + + + Another method is by using anonymous Python. + Here is an example: + + python () { + if condition == value: + d.setVar('VARIABLE', 'myclass') + else: + d.setVar('VARIABLE', '') + } + + + + + Alternatively, you could use an in-line Python expression + in the following form: + + inherit ${@'classname' if condition else ''} + inherit ${@functionname(params)} + + In all cases, if the expression evaluates to an empty + string, the statement does not trigger a syntax error + because it becomes a no-op. + +
+ +
+ <filename>include</filename> Directive + + + BitBake understands the include + directive. + This directive causes BitBake to parse whatever file you specify, + and to insert that file at that location. + The directive is much like its equivalent in Make except + that if the path specified on the include line is a relative + path, BitBake locates the first file it can find + within BBPATH. + + + + As an example, suppose you needed a recipe to include some + self-test definitions: + + include test_defs.inc + + + The include directive does not + produce an error when the file cannot be found. + Consequently, it is recommended that if the file you + are including is expected to exist, you should use + require + instead of include. + Doing so makes sure that an error is produced if the + file cannot be found. + + +
+ +
+ <filename>require</filename> Directive + + + BitBake understands the require + directive. + This directive behaves just like the + include directive with the exception that + BitBake raises a parsing error if the file to be included cannot + be found. + Thus, any file you require is inserted into the file that is + being parsed at the location of the directive. + + + + Similar to how BitBake handles + include, + if the path specified + on the require line is a relative path, BitBake locates + the first file it can find within BBPATH. + + + + As an example, suppose you have two versions of a recipe + (e.g. foo_1.2.2.bb and + foo_2.0.0.bb) where + each version contains some identical functionality that could be + shared. + You could create an include file named foo.inc + that contains the common definitions needed to build "foo". + You need to be sure foo.inc is located in the + same directory as your two recipe files as well. + Once these conditions are set up, you can share the functionality + using a require directive from within each + recipe: + + require foo.inc + + +
+ +
+ <filename>INHERIT</filename> Configuration Directive + + + When creating a configuration file (.conf), + you can use the INHERIT directive to + inherit a class. + BitBake only supports this directive when used within + a configuration file. + + + + As an example, suppose you needed to inherit a class + file called abc.bbclass from a + configuration file as follows: + + INHERIT += "abc" + + This configuration directive causes the named + class to be inherited at the point of the directive + during parsing. + As with the inherit directive, the + .bbclass file must be located in a + "classes" subdirectory in one of the directories specified + in BBPATH. + + Because .conf files are parsed + first during BitBake's execution, using + INHERIT to inherit a class effectively + inherits the class globally (i.e. for all recipes). + + If you want to use the directive to inherit + multiple classes, you can provide them on the same line in the + local.conf file. + Use spaces to separate the classes. + The following example shows how to inherit both the + autotools and pkgconfig + classes: + + inherit autotools pkgconfig + + +
+
+ +
+ Functions + + + As with most languages, functions are the building blocks that + are used to build up operations into tasks. + BitBake supports these types of functions: + + Shell Functions: + Functions written in shell script and executed either + directly as functions, tasks, or both. + They can also be called by other shell functions. + + BitBake Style Python Functions: + Functions written in Python and executed by BitBake or other + Python functions using bb.build.exec_func(). + + Python Functions: + Functions written in Python and executed by Python. + + Anonymous Python Functions: + Python functions executed automatically during + parsing. + + + Regardless of the type of function, you can only + define them in class (.bbclass) + and recipe (.bb or .inc) + files. + + +
+ Shell Functions + + + Functions written in shell script and executed either + directly as functions, tasks, or both. + They can also be called by other shell functions. + Here is an example shell function definition: + + some_function () { + echo "Hello World" + } + + When you create these types of functions in your recipe + or class files, you need to follow the shell programming + rules. + The scripts are executed by /bin/sh, + which may not be a bash shell but might be something + such as dash. + You should not use Bash-specific script (bashisms). + +
+ +
+ BitBake Style Python Functions + + + These functions are written in Python and executed by + BitBake or other Python functions using + bb.build.exec_func(). + + + + An example BitBake function is: + + python some_python_function () { + d.setVar("TEXT", "Hello World") + print d.getVar("TEXT", True) + } + + Because the Python "bb" and "os" modules are already + imported, you do not need to import these modules. + Also in these types of functions, the datastore ("d") + is a global variable and is always automatically + available. + + + + Variable expressions (e.g. ${X}) are no + longer expanded within Python functions. + This behavior is intentional in order to allow you to freely + set variable values to expandable expressions without having + them expanded prematurely. + If you do wish to expand a variable within a Python function, + use d.getVar("X", True). + Or, for more complicated expressions, use + d.expand(). + +
+ +
+ Python Functions + + + These functions are written in Python and are executed by + other Python code. + Examples of Python functions are utility functions + that you intend to call from in-line Python or + from within other Python functions. + Here is an example: + + def get_depends(d): + if d.getVar('SOMECONDITION', True): + return "dependencywithcond" + else: + return "dependency" + SOMECONDITION = "1" + DEPENDS = "${@get_depends(d)}" + + This would result in DEPENDS + containing dependencywithcond. + + + + Here are some things to know about Python functions: + + Python functions can take parameters. + + The BitBake datastore is not + automatically available. + Consequently, you must pass it in as a + parameter to the function. + + The "bb" and "os" Python modules are + automatically available. + You do not need to import them. + + + +
+ +
+ Anonymous Python Functions + + + Sometimes it is useful to run some code during + parsing to set variables or to perform other operations + programmatically. + To do this, you can define an anonymous Python function. + Here is an example that conditionally sets a + variable based on the value of another variable: + + python __anonymous () { + if d.getVar('SOMEVAR', True) == 'value': + d.setVar('ANOTHERVAR', 'value2') + } + + The "__anonymous" function name is optional, so the + following example is functionally equivalent to the above: + + python () { + if d.getVar('SOMEVAR', True) == 'value': + d.setVar('ANOTHERVAR', 'value2') + } + + Because unlike other Python functions anonymous + Python functions are executed during parsing, the + "d" variable within an anonymous Python function represents + the datastore for the entire recipe. + Consequently, you can set variable values here and + those values can be picked up by other functions. + +
+ +
+ Flexible Inheritance for Class Functions + + + Through coding techniques and the use of + EXPORT_FUNCTIONS, BitBake supports + exporting a function from a class such that the + class function appears as the default implementation + of the function, but can still be called if a recipe + inheriting the class needs to define its own version of + the function. + + + + To understand the benefits of this feature, consider + the basic scenario where a class defines a task function + and your recipe inherits the class. + In this basic scenario, your recipe inherits the task + function as defined in the class. + If desired, your recipe can add to the start and end of the + function by using the "_prepend" or "_append" operations + respectively, or it can redefine the function completely. + However, if it redefines the function, there is + no means for it to call the class version of the function. + EXPORT_FUNCTIONS provides a mechanism + that enables the recipe's version of the function to call + the original version of the function. + + + + To make use of this technique, you need the following + things in place: + + + The class needs to define the function as follows: + + classname_functionname + + For example, if you have a class file + bar.bbclass and a function named + do_foo, the class must define the function + as follows: + + bar_do_foo + + + + The class needs to contain the EXPORT_FUNCTIONS + statement as follows: + + EXPORT_FUNCTIONS functionname + + For example, continuing with the same example, the + statement in the bar.bbclass would be + as follows: + + EXPORT_FUNCTIONS do_foo + + + + You need to call the function appropriately from within your + recipe. + Continuing with the same example, if your recipe + needs to call the class version of the function, + it should call bar_do_foo. + Assuming do_foo was a shell function + and EXPORT_FUNCTIONS was used as above, + the recipe's function could conditionally call the + class version of the function as follows: + + do_foo() { + if [ somecondition ] ; then + bar_do_foo + else + # Do something else + fi + } + + To call your modified version of the function as defined + in your recipe, call it as do_foo. + + + With these conditions met, your single recipe + can freely choose between the original function + as defined in the class file and the modified function in your recipe. + If you do not set up these conditions, you are limited to using one function + or the other. + +
+
+ +
+ Tasks + + + Tasks are BitBake execution units that originate as + functions and make up the steps that BitBake needs to run + for given recipe. + Tasks are only supported in recipe (.bb + or .inc) and class + (.bbclass) files. + By convention, task names begin with the string "do_". + + + + Here is an example of a task that prints out the date: + + python do_printdate () { + import time + print time.strftime('%Y%m%d', time.gmtime()) + } + addtask printdate after do_fetch before do_build + + + +
+ Promoting a Function to a Task + + + Any function can be promoted to a task by applying the + addtask command. + The addtask command also describes + inter-task dependencies. + Here is the function from the previous section but with the + addtask command promoting it to a task + and defining some dependencies: + + python do_printdate () { + import time + print time.strftime('%Y%m%d', time.gmtime()) + } + addtask printdate after do_fetch before do_build + + In the example, the function is defined and then promoted + as a task. + The do_printdate task becomes a dependency of + the do_build task, which is the default + task. + And, the do_printdate task is dependent upon + the do_fetch task. + Execution of the do_build task results + in the do_printdate task running first. + +
+ +
+ Deleting a Task + + + As well as being able to add tasks, you can delete them. + Simply use the deltask command to + delete a task. + For example, to delete the example task used in the previous + sections, you would use: + + deltask printdate + + If you delete a task using the deltask + command and the task has dependencies, the dependencies are + not reconnected. + For example, suppose you have three tasks named + do_a, do_b, and + do_c. + Furthermore, do_c is dependent on + do_b, which in turn is dependent on + do_a. + Given this scenario, if you use deltask + to delete do_b, the implicit dependency + relationship between do_c and + do_a through do_b + no longer exists, and do_c dependencies + are not updated to include do_a. + Thus, do_c is free to run before + do_a. + + + + If you want dependencies such as these to remain intact, use + the noexec varflag to disable the task + instead of using the deltask command to + delete it: + + do_b[noexec] = "1" + + +
+ +
+ Passing Information Into the Build Task Environment + + + When running a task, BitBake tightly controls the shell execution + environment of the build tasks to make + sure unwanted contamination from the build machine cannot + influence the build. + + By default, BitBake cleans the environment to include only those + things exported or listed in its whitelist to ensure that the build + environment is reproducible and consistent. + You can prevent this "cleaning" by setting the + BB_PRESERVE_ENV + variable. + + Consequently, if you do want something to get passed into the + build task environment, you must take these two steps: + + + Tell BitBake to load what you want from the environment + into the datastore. + You can do so through the + BB_ENV_WHITELIST + and + BB_ENV_EXTRAWHITE + variables. + For example, assume you want to prevent the build system from + accessing your $HOME/.ccache + directory. + The following command "whitelists" the environment variable + CCACHE_DIR causing BitBack to allow that + variable into the datastore: + + export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE CCACHE_DIR" + + + Tell BitBake to export what you have loaded into the + datastore to the task environment of every running task. + Loading something from the environment into the datastore + (previous step) only makes it available in the datastore. + To export it to the task environment of every running task, + use a command similar to the following in your local configuration + file local.conf or your + distribution configuration file: + + export CCACHE_DIR + + + A side effect of the previous steps is that BitBake + records the variable as a dependency of the build process + in things like the setscene checksums. + If doing so results in unnecessary rebuilds of tasks, you can + whitelist the variable so that the setscene code + ignores the dependency when it creates checksums. + + + + + + Sometimes, it is useful to be able to obtain information + from the original execution environment. + Bitbake saves a copy of the original environment into + a special variable named + BB_ORIGENV. + + + + The BB_ORIGENV variable returns a datastore + object that can be queried using the standard datastore operators + such as getVar(, False). + The datastore object is useful, for example, to find the original + DISPLAY variable. + Here is an example: + + origenv = d.getVar("BB_ORIGENV", False) + bar = origenv.getVar("BAR", False) + + The previous example returns BAR from the original + execution environment. + +
+
+ +
+ Variable Flags + + + Variable flags (varflags) help control a task's functionality + and dependencies. + BitBake reads and writes varflags to the datastore using the following + command forms: + + variable = d.getVarFlags("variable") + self.d.setVarFlags("FOO", {"func": True}) + + + + + When working with varflags, the same syntax, with the exception of + overrides, applies. + In other words, you can set, append, and prepend varflags just like + variables. + See the + "Variable Flag Syntax" + section for details. + + + + BitBake has a defined set of varflags available for recipes and + classes. + Tasks support a number of these flags which control various + functionality of the task: + + cleandirs: + Empty directories that should created before the task runs. + + depends: + Controls inter-task dependencies. + See the + DEPENDS + variable and the + "Inter-Task Dependencies" + section for more information. + + deptask: + Controls task build-time dependencies. + See the + DEPENDS + variable and the + "Build Dependencies" + section for more information. + + dirs: + Directories that should be created before the task runs. + The last directory listed will be used as the work directory + for the task. + + lockfiles: + Specifies one or more lockfiles to lock while the task + executes. + Only one task may hold a lockfile, and any task that + attempts to lock an already locked file will block until + the lock is released. + You can use this variable flag to accomplish mutual + exclusion. + + noexec: + Marks the tasks as being empty and no execution required. + The noexec flag can be used to set up + tasks as dependency placeholders, or to disable tasks defined + elsewhere that are not needed in a particular recipe. + + nostamp: + Tells BitBake to not generate a stamp file for a task, + which implies the task should always be executed. + + postfuncs: + List of functions to call after the completion of the task. + + prefuncs: + List of functions to call before the task executes. + + rdepends: + Controls inter-task runtime dependencies. + See the + RDEPENDS + variable, the + RRECOMMENDS + variable, and the + "Inter-Task Dependencies" + section for more information. + + rdeptask: + Controls task runtime dependencies. + See the + RDEPENDS + variable, the + RRECOMMENDS + variable, and the + "Runtime Dependencies" + section for more information. + + recideptask: + When set in conjunction with + recrdeptask, specifies a task that + should be inspected for additional dependencies. + + recrdeptask: + Controls task recursive runtime dependencies. + See the + RDEPENDS + variable, the + RRECOMMENDS + variable, and the + "Recursive Dependencies" + section for more information. + + stamp-extra-info: + Extra stamp information to append to the task's stamp. + As an example, OpenEmbedded uses this flag to allow + machine-specific tasks. + + umask: + The umask to run the task under. + + + + + + Several varflags are useful for controlling how signatures are + calculated for variables. + For more information on this process, see the + "Checksums (Signatures)" + section. + + vardeps: + Specifies a space-separated list of additional + variables to add to a variable's dependencies + for the purposes of calculating its signature. + Adding variables to this list is useful, for example, when + a function refers to a variable in a manner that + does not allow BitBake to automatically determine + that the variable is referred to. + + vardepsexclude: + Specifies a space-separated list of variables + that should be excluded from a variable's dependencies + for the purposes of calculating its signature. + + vardepvalue: + If set, instructs BitBake to ignore the actual + value of the variable and instead use the specified + value when calculating the variable's signature. + + vardepvalueexclude: + Specifies a pipe-separated list of strings to exclude + from the variable's value when calculating the + variable's signature. + + + +
+ +
+ Events + + + BitBake allows installation of event handlers within + recipe and class files. + Events are triggered at certain points during operation, + such as the beginning of an operation against a given recipe + (*.bb file), the start of a given task, + task failure, task success, and so forth. + The intent is to make it easy to do things like email + notification on build failure. + + + + Following is an example event handler that + prints the name of the event and the content of + the FILE variable: + + addhandler myclass_eventhandler + python myclass_eventhandler() { + from bb.event import getName + from bb import data + print("The name of the Event is %s" % getName(e)) + print("The file we run for is %s" % data.getVar('FILE', e.data, True)) + } + + This event handler gets called every time an event is + triggered. + A global variable "e" is defined and + "e.data" contains an instance of + "bb.data". + With the getName(e) method, one can get + the name of the triggered event. + + + + Because you probably are only interested in a subset of events, + you would likely use the [eventmask] flag + for your event handler to be sure that only certain events + trigger the handler. + Given the previous example, suppose you only wanted the + bb.build.TaskFailed event to trigger that + event handler. + Use the flag as follows: + + addhandler myclass_eventhandler + myclass_eventhandler[eventmask] = "bb.build.TaskFailed" + python myclass_eventhandler() { + from bb.event import getName + from bb import data + print("The name of the Event is %s" % getName(e)) + print("The file we run for is %s" % data.getVar('FILE', e.data, True)) + } + + + + + During a standard build, the following common events might occur: + + + bb.event.ConfigParsed() + + + bb.event.ParseStarted() + + + bb.event.ParseProgress() + + + bb.event.ParseCompleted() + + + bb.event.BuildStarted() + + + bb.build.TaskStarted() + + + bb.build.TaskInvalid() + + + bb.build.TaskFailedSilent() + + + bb.build.TaskFailed() + + + bb.build.TaskSucceeded() + + + bb.event.BuildCompleted() + + + bb.cooker.CookerExit() + + + Here is a list of other events that occur based on specific requests + to the server: + + + bb.event.TreeDataPreparationStarted() + + + bb.event.TreeDataPreparationProgress + + + bb.event.TreeDataPreparationCompleted + + + bb.event.DepTreeGenerated + + + bb.event.CoreBaseFilesFound + + + bb.event.ConfigFilePathFound + + + bb.event.FilesMatchingFound + + + bb.event.ConfigFilesFound + + + bb.event.TargetsTreeGenerated + + + +
+ +
+ Variants - Class Extension Mechanism + + + BitBake supports two features that facilitate creating + from a single recipe file multiple incarnations of that + recipe file where all incarnations are buildable. + These features are enabled through the + BBCLASSEXTEND + and + BBVERSIONS + variables. + + The mechanism for this class extension is extremely + specific to the implementation. + Usually, the recipe's + PROVIDES, + PN, and + DEPENDS + variables would need to be modified by the extension class. + For specific examples, see the OE-Core + native, nativesdk, + and multilib classes. + + + BBCLASSEXTEND: + This variable is a space separated list of classes used to "extend" the + recipe for each variant. + Here is an example that results in a second incarnation of the current + recipe being available. + This second incarnation will have the "native" class inherited. + + BBCLASSEXTEND = "native" + + BBVERSIONS: + This variable allows a single recipe to build multiple versions of a + project from a single recipe file. + You can also specify conditional metadata + (using the + OVERRIDES + mechanism) for a single version, or an optionally named range of versions. + Here is an example: + + BBVERSIONS = "1.0 2.0 git" + SRC_URI_git = "git://someurl/somepath.git" + + BBVERSIONS = "1.0.[0-6]:1.0.0+ \ 1.0.[7-9]:1.0.7+" + SRC_URI_append_1.0.7+ = "file://some_patch_which_the_new_versions_need.patch;patch=1" + + The name of the range defaults to the original version of the + recipe. + For example, in OpenEmbedded, the recipe file + foo_1.0.0+.bb creates a default name range + of 1.0.0+. + This is useful because the range name is not only placed + into overrides, but it is also made available for the metadata to use + in the variable that defines the base recipe versions for use in + file:// search paths + (FILESPATH). + + + +
+ +
+ Dependencies + + + To allow for efficient operation given multiple processes + executing in parallel, BitBake handles dependencies at + the task level. + BitBake supports a robust method to handle these dependencies. + + + + This section describes several types of dependency mechanisms. + + +
+ Dependencies Internal to the <filename>.bb</filename> File + + + BitBake uses the addtask directive + to manage dependencies that are internal to a given recipe + file. + You can use the addtask directive to + indicate when a task is dependent on other tasks or when + other tasks depend on that recipe. + Here is an example: + + addtask printdate after do_fetch before do_build + + In this example, the printdate task is + depends on the completion of the do_fetch + task. + And, the do_build depends on the completion + of the printdate task. + +
+ +
+ Build Dependencies + + + BitBake uses the + DEPENDS + variable to manage build time dependencies. + The "deptask" varflag for tasks signifies the task of each + item listed in DEPENDS that must + complete before that task can be executed. + Here is an example: + + do_configure[deptask] = "do_populate_sysroot" + + In this example, the do_populate_sysroot + task of each item in DEPENDS must complete before + do_configure can execute. + +
+ +
+ Runtime Dependencies + + + BitBake uses the + PACKAGES, + RDEPENDS, and + RRECOMMENDS + variables to manage runtime dependencies. + + + + The PACKAGES variable lists runtime + packages. + Each of those packages can have RDEPENDS and + RRECOMMENDS runtime dependencies. + The "rdeptask" flag for tasks is used to signify the task of each + item runtime dependency which must have completed before that + task can be executed. + + do_package_qa[rdeptask] = "do_packagedata" + + In the previous example, the do_packagedata + task of each item in RDEPENDS must have + completed before do_package_qa can execute. + +
+ +
+ Recursive Dependencies + + + BitBake uses the "recrdeptask" flag to manage + recursive task dependencies. + BitBake looks through the build-time and runtime + dependencies of the current recipe, looks through + the task's inter-task + dependencies, and then adds dependencies for the + listed task. + Once BitBake has accomplished this, it recursively works through + the dependencies of those tasks. + Iterative passes continue until all dependencies are discovered + and added. + + + + You might want to not only have BitBake look for + dependencies of those tasks, but also have BitBake look + for build-time and runtime dependencies of the dependent + tasks as well. + If that is the case, you need to reference the task name + itself in the task list: + + do_a[recrdeptask] = "do_a do_b" + + +
+ +
+ Inter-Task Dependencies + + + BitBake uses the "depends" flag in a more generic form + to manage inter-task dependencies. + This more generic form allows for inter-dependency + checks for specific tasks rather than checks for + the data in DEPENDS. + Here is an example: + + do_patch[depends] = "quilt-native:do_populate_sysroot" + + In this example, the do_populate_sysroot + task of the target quilt-native + must have completed before the + do_patch task can execute. + + + + The "rdepends" flag works in a similar way but takes targets + in the runtime namespace instead of the build-time dependency + namespace. + +
+
+ +
+ Accessing Datastore Variables Using Python + + + It is often necessary to access variables in the + BitBake datastore using Python functions. + The Bitbake datastore has an API that allows you this + access. + Here is a list of available operations: + + + + + + + + + + Operation + Description + + + + + d.getVar("X", expand=False) + Returns the value of variable "X". + Using "expand=True" expands the value. + + + d.setVar("X", "value") + Sets the variable "X" to "value". + + + d.appendVar("X", "value") + Adds "value" to the end of the variable "X". + + + d.prependVar("X", "value") + Adds "value" to the start of the variable "X". + + + d.delVar("X") + Deletes the variable "X" from the datastore. + + + d.renameVar("X", "Y") + Renames the variable "X" to "Y". + + + d.getVarFlag("X", flag, expand=False) + Gets then named flag from the variable "X". + Using "expand=True" expands the named flag. + + + d.setVarFlag("X", flag, "value") + Sets the named flag for variable "X" to "value". + + + d.appendVarFlag("X", flag, "value") + Appends "value" to the named flag on the + variable "X". + + + d.prependVarFlag("X", flag, "value") + Prepends "value" to the named flag on + the variable "X". + + + d.delVarFlag("X", flag) + Deletes the named flag on the variable + "X" from the datastore. + + + d.setVarFlags("X", flagsdict) + Sets the flags specified in + the flagsdict() parameter. + setVarFlags does not clear previous flags. + Think of this operation as addVarFlags. + + + d.getVarFlags("X") + Returns a flagsdict of the flags for + the variable "X". + + + d.delVarFlags("X") + Deletes all the flags for the variable "X". + + + d.expand(expression) + Expands variable references in the specified string expression. + + + + + +
+ +
+ Task Checksums and Setscene + + + BitBake uses checksums (or signatures) along with the setscene + to determine if a task needs to be run. + This section describes the process. + To help understand how BitBake does this, the section assumes an + OpenEmbedded metadata-based example. + + + + This list is a place holder of content existed from previous work + on the manual. + Some or all of it probably needs integrated into the subsections + that make up this section. + For now, I have just provided a short glossary-like description + for each variable. + Ultimately, this list goes away. + + STAMP: + The base path to create stamp files. + STAMPCLEAN + Again, the base path to create stamp files but can use wildcards + for matching a range of files for clean operations. + + BB_STAMP_WHITELIST + Lists stamp files that are looked at when the stamp policy + is "whitelist". + + BB_STAMP_POLICY + Defines the mode for comparing timestamps of stamp files. + + BB_HASHCHECK_FUNCTION + Specifies the name of the function to call during + the "setscene" part of the task's execution in order + to validate the list of task hashes. + + BB_SETSCENE_VERIFY_FUNCTION + Specifies a function to call that verifies the list of + planned task execution before the main task execution + happens. + + BB_SETSCENE_DEPVALID + Specifies a function BitBake calls that determines + whether BitBake requires a setscene dependency to + be met. + + BB_TASKHASH + Within an executing task, this variable holds the hash + of the task as returned by the currently enabled + signature generator. + + + +
+ diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml new file mode 100644 index 000000000..ae7e4cee8 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml @@ -0,0 +1,2319 @@ + %poky; ] > + + + + +Variables Glossary + + + This chapter lists common variables used by BitBake and gives an overview + of their function and contents. + + + + Following are some points regarding the variables listed in this glossary: + + The variables listed in this glossary + are specific to BitBake. + Consequently, the descriptions are limited to that context. + + Also, variables exist in other systems that use BitBake + (e.g. The Yocto Project and OpenEmbedded) that have names identical + to those found in this glossary. + For such cases, the variables in those systems extend the + functionality of the variable as it is described here in + this glossary. + + Finally, there are variables mentioned in this + glossary that do not appear in the BitBake glossary. + These other variables are variables used in systems that use + BitBake. + + + + + + + + A + B + C + D + E + F + G + H + + + + L + M + + O + P + + R + S + T + + + + + + + + + A + + ASSUME_PROVIDED + + + Lists recipe names + (PN + values) BitBake does not attempt to build. + Instead, BitBake assumes these recipes have already been + built. + + + + In OpenEmbedded Core, ASSUME_PROVIDED + mostly specifies native tools that should not be built. + An example is git-native, which + when specified allows for the Git binary from the host to + be used rather than building + git-native. + + + + + + + + B + + B + + + The directory in which BitBake executes functions + during a recipe's build process. + + + + + BB_ALLOWED_NETWORKS + + + Specifies a space-delimited list of hosts that the fetcher + is allowed to use to obtain the required source code. + Following are considerations surrounding this variable: + + + This host list is only used if + BB_NO_NETWORK + is either not set or set to "0". + + + Limited support for wildcard matching against the + beginning of host names exists. + For example, the following setting matches + git.gnu.org, + ftp.gnu.org, and + foo.git.gnu.org. + + BB_ALLOWED_NETWORKS = "*.gnu.org" + + + + Mirrors not in the host list are skipped and + logged in debug. + + + Attempts to access networks not in the host list + cause a failure. + + + Using BB_ALLOWED_NETWORKS in + conjunction with + PREMIRRORS + is very useful. + Adding the host you want to use to + PREMIRRORS results in the source code + being fetched from an allowed location and avoids raising + an error when a host that is not allowed is in a + SRC_URI + statement. + This is because the fetcher does not attempt to use the + host listed in SRC_URI after a + successful fetch from the + PREMIRRORS occurs. + + + + + BB_CONSOLELOG + + + Specifies the path to a log file into which BitBake's user + interface writes output during the build. + + + + + BB_CURRENTTASK + + + Contains the name of the currently running task. + The name does not include the + do_ prefix. + + + + + BB_DANGLINGAPPENDS_WARNONLY + + + Defines how BitBake handles situations where an append + file (.bbappend) has no + corresponding recipe file (.bb). + This condition often occurs when layers get out of sync + (e.g. oe-core bumps a + recipe version and the old recipe no longer exists and the + other layer has not been updated to the new version + of the recipe yet). + + + + The default fatal behavior is safest because it is + the sane reaction given something is out of sync. + It is important to realize when your changes are no longer + being applied. + + + + + BB_DEFAULT_TASK + + + The default task to use when none is specified (e.g. + with the -c command line option). + The task name specified should not include the + do_ prefix. + + + + + BB_DISKMON_DIRS + + + Monitors disk space and available inodes during the build + and allows you to control the build based on these + parameters. + + + + Disk space monitoring is disabled by default. + When setting this variable, use the following form: + + BB_DISKMON_DIRS = "<action>,<dir>,<threshold> [...]" + + where: + + <action> is: + ABORT: Immediately abort the build when + a threshold is broken. + STOPTASKS: Stop the build after the currently + executing tasks have finished when + a threshold is broken. + WARN: Issue a warning but continue the + build when a threshold is broken. + Subsequent warnings are issued as + defined by the + BB_DISKMON_WARNINTERVAL variable, + which must be defined. + + <dir> is: + Any directory you choose. You can specify one or + more directories to monitor by separating the + groupings with a space. If two directories are + on the same device, only the first directory + is monitored. + + <threshold> is: + Either the minimum available disk space, + the minimum number of free inodes, or + both. You must specify at least one. To + omit one or the other, simply omit the value. + Specify the threshold using G, M, K for Gbytes, + Mbytes, and Kbytes, respectively. If you do + not specify G, M, or K, Kbytes is assumed by + default. Do not use GB, MB, or KB. + + + + + Here are some examples: + + BB_DISKMON_DIRS = "ABORT,${TMPDIR},1G,100K WARN,${SSTATE_DIR},1G,100K" + BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},1G" + BB_DISKMON_DIRS = "ABORT,${TMPDIR},,100K" + + The first example works only if you also set + the BB_DISKMON_WARNINTERVAL variable. + This example causes the build system to immediately + abort when either the disk space in ${TMPDIR} drops + below 1 Gbyte or the available free inodes drops below + 100 Kbytes. + Because two directories are provided with the variable, the + build system also issues a + warning when the disk space in the + ${SSTATE_DIR} directory drops + below 1 Gbyte or the number of free inodes drops + below 100 Kbytes. + Subsequent warnings are issued during intervals as + defined by the BB_DISKMON_WARNINTERVAL + variable. + + + + The second example stops the build after all currently + executing tasks complete when the minimum disk space + in the ${TMPDIR} + directory drops below 1 Gbyte. + No disk monitoring occurs for the free inodes in this case. + + + + The final example immediately aborts the build when the + number of free inodes in the ${TMPDIR} directory + drops below 100 Kbytes. + No disk space monitoring for the directory itself occurs + in this case. + + + + + BB_DISKMON_WARNINTERVAL + + + Defines the disk space and free inode warning intervals. + + + + If you are going to use the + BB_DISKMON_WARNINTERVAL variable, you must + also use the + BB_DISKMON_DIRS variable + and define its action as "WARN". + During the build, subsequent warnings are issued each time + disk space or number of free inodes further reduces by + the respective interval. + + + + If you do not provide a BB_DISKMON_WARNINTERVAL + variable and you do use BB_DISKMON_DIRS with + the "WARN" action, the disk monitoring interval defaults to + the following: + + BB_DISKMON_WARNINTERVAL = "50M,5K" + + + + + When specifying the variable in your configuration file, + use the following form: + + BB_DISKMON_WARNINTERVAL = "<disk_space_interval>,<disk_inode_interval>" + + where: + + <disk_space_interval> is: + An interval of memory expressed in either + G, M, or K for Gbytes, Mbytes, or Kbytes, + respectively. You cannot use GB, MB, or KB. + + <disk_inode_interval> is: + An interval of free inodes expressed in either + G, M, or K for Gbytes, Mbytes, or Kbytes, + respectively. You cannot use GB, MB, or KB. + + + + + Here is an example: + + BB_DISKMON_DIRS = "WARN,${SSTATE_DIR},1G,100K" + BB_DISKMON_WARNINTERVAL = "50M,5K" + + These variables cause BitBake to + issue subsequent warnings each time the available + disk space further reduces by 50 Mbytes or the number + of free inodes further reduces by 5 Kbytes in the + ${SSTATE_DIR} directory. + Subsequent warnings based on the interval occur each time + a respective interval is reached beyond the initial warning + (i.e. 1 Gbytes and 100 Kbytes). + + + + + BB_ENV_WHITELIST + + + Specifies the internal whitelist of variables to allow + through from the external environment into BitBake's + datastore. + If the value of this variable is not specified + (which is the default), the following list is used: + BBPATH, + BB_PRESERVE_ENV, + BB_ENV_WHITELIST, + and + BB_ENV_EXTRAWHITE. + + You must set this variable in the external environment + in order for it to work. + + + + + + BB_ENV_EXTRAWHITE + + + Specifies an additional set of variables to allow through + (whitelist) from the external environment into BitBake's + datastore. + This list of variables are on top of the internal list + set in + BB_ENV_WHITELIST. + + You must set this variable in the external + environment in order for it to work. + + + + + + BB_FETCH_PREMIRRORONLY + + + When set to "1", causes BitBake's fetcher module to only + search + PREMIRRORS + for files. + BitBake will not search the main + SRC_URI + or + MIRRORS. + + + + + BB_FILENAME + + + Contains the filename of the recipe that owns the currently + running task. + For example, if the do_fetch task that + resides in the my-recipe.bb is + executing, the BB_FILENAME variable + contains "/foo/path/my-recipe.bb". + + + + + BB_GENERATE_MIRROR_TARBALLS + + + Causes tarballs of the Git repositories, including the + Git metadata, to be placed in the + DL_DIR + directory. + Anyone wishing to create a source mirror would want to + enable this variable. + + + + For performance reasons, creating and placing tarballs of + the Git repositories is not the default action by BitBake. + + BB_GENERATE_MIRROR_TARBALLS = "1" + + + + + + BB_HASHCONFIG_WHITELIST + + + Lists variables that are excluded from base configuration + checksum, which is used to determine if the cache can + be reused. + + + + One of the ways BitBake determines whether to re-parse the + main metadata is through checksums of the variables in the + datastore of the base configuration data. + There are variables that you typically want to exclude when + checking whether or not to re-parse and thus rebuild the + cache. + As an example, you would usually exclude + TIME and DATE + because these variables are always changing. + If you did not exclude them, BitBake would never reuse the + cache. + + + + + BB_HASHBASE_WHITELIST + + + Lists variables that are excluded from checksum and + dependency data. + Variables that are excluded can therefore change without + affecting the checksum mechanism. + A common example would be the variable for the path of + the build. + BitBake's output should not (and usually does not) depend + on the directory in which it was built. + + + + + BB_HASHCHECK_FUNCTION + + + Specifies the name of the function to call during the + "setscene" part of the task's execution in order to + validate the list of task hashes. + The function returns the list of setscene tasks that should + be executed. + + + + At this point in the execution of the code, the objective + is to quickly verify if a given setscene function is likely + to work or not. + It's easier to check the list of setscene functions in + one pass than to call many individual tasks. + The returned list need not be completely accurate. + A given setscene task can still later fail. + However, the more accurate the data returned, the more + efficient the build will be. + + + + + BB_INVALIDCONF + + + Used in combination with the + ConfigParsed event to trigger + re-parsing the base metadata (i.e. all the + recipes). + The ConfigParsed event can set the + variable to trigger the re-parse. + You must be careful to avoid recursive loops with this + functionality. + + + + + BB_LOGFMT + + + Specifies the name of the log files saved into + ${T}. + By default, the BB_LOGFMT variable + is undefined and the log file names get created using the + following form: + + log.{task}.{pid} + + If you want to force log files to take a specific name, + you can set this variable in a configuration file. + + + + + BB_NICE_LEVEL + + + Allows BitBake to run at a specific priority + (i.e. nice level). + System permissions usually mean that BitBake can reduce its + priority but not raise it again. + See + BB_TASK_NICE_LEVEL + for additional information. + + + + + BB_NO_NETWORK + + + Disables network access in the BitBake fetcher modules. + With this access disabled, any command that attempts to + access the network becomes an error. + + + + Disabling network access is useful for testing source + mirrors, running builds when not connected to the Internet, + and when operating in certain kinds of firewall + environments. + + + + + BB_NUMBER_THREADS + + + The maximum number of tasks BitBake should run in parallel + at any one time. + If your host development system supports multiple cores, + a good rule of thumb is to set this variable to twice the + number of cores. + + + + + BB_NUMBER_PARSE_THREADS + + + Sets the number of threads BitBake uses when parsing. + By default, the number of threads is equal to the number + of cores on the system. + + + + + BB_ORIGENV + + + Contains a copy of the original external environment in + which BitBake was run. + The copy is taken before any whitelisted variable values + are filtered into BitBake's datastore. + + The contents of this variable is a datastore object + that can be queried using the normal datastore + operations. + + + + + + BB_PRESERVE_ENV + + + Disables whitelisting and instead allows all variables + through from the external environment into BitBake's + datastore. + + You must set this variable in the external + environment in order for it to work. + + + + + + BB_RUNFMT + + + Specifies the name of the executable script files + (i.e. run files) saved into + ${T}. + By default, the BB_RUNFMT variable + is undefined and the run file names get created using the + following form: + + run.{task}.{pid} + + If you want to force run files to take a specific name, + you can set this variable in a configuration file. + + + + + BB_RUNTASK + + + Contains the name of the currently executing task. + The value does not include the "do_" prefix. + For example, if the currently executing task is + do_config, the value is + "config". + + + + + BB_SCHEDULER + + + Selects the name of the scheduler to use for the + scheduling of BitBake tasks. + Three options exist: + + basic - + The basic framework from which everything derives. + Using this option causes tasks to be ordered + numerically as they are parsed. + + speed - + Executes tasks first that have more tasks + depending on them. + The "speed" option is the default. + + completion - + Causes the scheduler to try to complete a given + recipe once its build has started. + + + + + + + BB_SCHEDULERS + + + Defines custom schedulers to import. + Custom schedulers need to be derived from the + RunQueueScheduler class. + + + + For information how to select a scheduler, see the + BB_SCHEDULER + variable. + + + + + BB_SETSCENE_DEPVALID + + + Specifies a function BitBake calls that determines + whether BitBake requires a setscene dependency to be met. + + + + When running a setscene task, BitBake needs to + know which dependencies of that setscene task also need + to be run. + Whether dependencies also need to be run is highly + dependent on the metadata. + The function specified by this variable returns a + "True" or "False" depending on whether the dependency needs + to be met. + + + + + BB_SETSCENE_VERIFY_FUNCTION + + + Specifies a function to call that verifies the list of + planned task execution before the main task execution + happens. + The function is called once BitBake has a list of setscene + tasks that have run and either succeeded or failed. + + + + The function allows for a task list check to see if they + make sense. + Even if BitBake was planning to skip a task, the + returned value of the function can force BitBake to run + the task, which is necessary under certain metadata + defined circumstances. + + + + + BB_SIGNATURE_EXCLUDE_FLAGS + + + Lists variable flags (varflags) + that can be safely excluded from checksum + and dependency data for keys in the datastore. + When generating checksum or dependency data for keys in the + datastore, the flags set against that key are normally + included in the checksum. + + + + For more information on varflags, see the + "Variable Flags" + section. + + + + + BB_SIGNATURE_HANDLER + + + Defines the name of the signature handler BitBake uses. + The signature handler defines the way stamp files are + created and handled, if and how the signature is + incorporated into the stamps, and how the signature + itself is generated. + + + + A new signature handler can be added by injecting a class + derived from the + SignatureGenerator class into the + global namespace. + + + + + BB_SRCREV_POLICY + + + Defines the behavior of the fetcher when it interacts with + source control systems and dynamic source revisions. + The BB_SRCREV_POLICY variable is + useful when working without a network. + + + + The variable can be set using one of two policies: + + cache - + Retains the value the system obtained previously + rather than querying the source control system + each time. + + clear - + Queries the source controls system every time. + With this policy, there is no cache. + The "clear" policy is the default. + + + + + + + BB_STAMP_POLICY + + + Defines the mode used for how timestamps of stamp files + are compared. + You can set the variable to one of the following modes: + + perfile - + Timestamp comparisons are only made + between timestamps of a specific recipe. + This is the default mode. + + full - + Timestamp comparisons are made for all + dependencies. + + whitelist - + Identical to "full" mode except timestamp + comparisons are made for recipes listed in the + BB_STAMP_WHITELIST + variable. + + + + Stamp policies are largely obsolete with the + introduction of setscene tasks. + + + + + + BB_STAMP_WHITELIST + + + Lists files whose stamp file timestamps are compared when + the stamp policy mode is set to "whitelist". + For information on stamp policies, see the + BB_STAMP_POLICY + variable. + + + + + BB_STRICT_CHECKSUM + + + Sets a more strict checksum mechanism for non-local URLs. + Setting this variable to a value causes BitBake + to report an error if it encounters a non-local URL + that does not have at least one checksum specified. + + + + + BB_TASK_IONICE_LEVEL + + + Allows adjustment of a task's Input/Output priority. + During Autobuilder testing, random failures can occur + for tasks due to I/O starvation. + These failures occur during various QEMU runtime timeouts. + You can use the BB_TASK_IONICE_LEVEL + variable to adjust the I/O priority of these tasks. + + This variable works similarly to the + BB_TASK_NICE_LEVEL + variable except with a task's I/O priorities. + + + + + Set the variable as follows: + + BB_TASK_IONICE_LEVEL = "class.prio" + + For class, the default value is + "2", which is a best effort. + You can use "1" for realtime and "3" for idle. + If you want to use realtime, you must have superuser + privileges. + + + + For prio, you can use any + value from "0", which is the highest priority, to "7", + which is the lowest. + The default value is "4". + You do not need any special privileges to use this range + of priority values. + + In order for your I/O priority settings to take effect, + you need the Completely Fair Queuing (CFQ) Scheduler + selected for the backing block device. + To select the scheduler, use the following command form + where device is the device + (e.g. sda, sdb, and so forth): + + $ sudo sh -c “echo cfq > /sys/block/device/queu/scheduler + + + + + + + BB_TASK_NICE_LEVEL + + + Allows specific tasks to change their priority + (i.e. nice level). + + + + You can use this variable in combination with task + overrides to raise or lower priorities of specific tasks. + For example, on the + Yocto Project + autobuilder, QEMU emulation in images is given a higher + priority as compared to build tasks to ensure that images + do not suffer timeouts on loaded systems. + + + + + BB_TASKHASH + + + Within an executing task, this variable holds the hash + of the task as returned by the currently enabled + signature generator. + + + + + BB_VERBOSE_LOGS + + + Controls how verbose BitBake is during builds. + If set, shell scripts echo commands and shell script output + appears on standard out (stdout). + + + + + BB_WORKERCONTEXT + + + Specifies if the current context is executing a task. + BitBake sets this variable to "1" when a task is + being executed. + The value is not set when the task is in server context + during parsing or event handling. + + + + + + BBCLASSEXTEND + + + Allows you to extend a recipe so that it builds variants + of the software. + Some examples of these variants for recipes from the + OpenEmbedded Core metadata are "natives" such as + quilt-native, which is a copy of + Quilt built to run on the build system; "crosses" such + as gcc-cross, which is a compiler + built to run on the build machine but produces binaries + that run on the target MACHINE; + "nativesdk", which targets the SDK machine instead of + MACHINE; and "mulitlibs" in the form + "multilib:multilib_name". + + + + To build a different variant of the recipe with a minimal + amount of code, it usually is as simple as adding the + variable to your recipe. + Here are two examples. + The "native" variants are from the OpenEmbedded Core + metadata: + + BBCLASSEXTEND =+ "native nativesdk" + BBCLASSEXTEND =+ "multilib:multilib_name" + + + + + + BBDEBUG + + + Sets the BitBake debug output level to a specific value + as incremented by the -d command line + option. + + You must set this variable in the external environment + in order for it to work. + + + + + + BBFILE_COLLECTIONS + + Lists the names of configured layers. + These names are used to find the other BBFILE_* + variables. + Typically, each layer appends its name to this variable in its + conf/layer.conf file. + + + + + BBFILE_PATTERN + + Variable that expands to match files from + BBFILES + in a particular layer. + This variable is used in the conf/layer.conf file and must + be suffixed with the name of the specific layer (e.g. + BBFILE_PATTERN_emenlow). + + + + BBFILE_PRIORITY + + Assigns the priority for recipe files in each layer. + This variable is useful in situations where the same recipe appears in + more than one layer. + Setting this variable allows you to prioritize a + layer against other layers that contain the same recipe - effectively + letting you control the precedence for the multiple layers. + The precedence established through this variable stands regardless of a + recipe's version + (PV variable). + For example, a layer that has a recipe with a higher PV value but for + which the BBFILE_PRIORITY is set to have a lower precedence still has a + lower precedence. + A larger value for the BBFILE_PRIORITY variable results in a higher + precedence. + For example, the value 6 has a higher precedence than the value 5. + If not specified, the BBFILE_PRIORITY variable is set based on layer + dependencies (see the + LAYERDEPENDS variable for + more information. + The default priority, if unspecified + for a layer with no dependencies, is the lowest defined priority + 1 + (or 1 if no priorities are defined). + + You can use the command bitbake-layers show-layers to list + all configured layers along with their priorities. + + + + + BBFILES + + List of recipe files BitBake uses to build software. + + + + BBINCLUDED + + + Contains a space-separated list of all of all files that + BitBake's parser included during parsing of the current + file. + + + + + BBINCLUDELOGS + + + If set to a value, enables printing the task log when + reporting a failed task. + + + + + BBINCLUDELOGS_LINES + + + If + BBINCLUDELOGS + is set, specifies the maximum number of lines from the + task log file to print when reporting a failed task. + If you do not set BBINCLUDELOGS_LINES, + the entire log is printed. + + + + + BBLAYERS + + Lists the layers to enable during the build. + This variable is defined in the bblayers.conf configuration + file in the build directory. + Here is an example: + + BBLAYERS = " \ + /home/scottrif/poky/meta \ + /home/scottrif/poky/meta-yocto \ + /home/scottrif/poky/meta-yocto-bsp \ + /home/scottrif/poky/meta-mykernel \ + " + + + This example enables four layers, one of which is a custom, user-defined layer + named meta-mykernel. + + + + + BBLAYERS_FETCH_DIR + + + Sets the base location where layers are stored. + By default, this location is set to + ${COREBASE}. + This setting is used in conjunction with + bitbake-layers layerindex-fetch and + tells bitbake-layers where to place + the fetched layers. + + + + + BBMASK + + + Prevents BitBake from processing recipes and recipe + append files. + + + + You can use the BBMASK variable + to "hide" these .bb and + .bbappend files. + BitBake ignores any recipe or recipe append files that + match any of the expressions. + It is as if BitBake does not see them at all. + Consequently, matching files are not parsed or otherwise + used by BitBake. + + The values you provide are passed to Python's regular + expression compiler. + The expressions are compared against the full paths to + the files. + For complete syntax information, see Python's + documentation at + . + + + + The following example uses a complete regular expression + to tell BitBake to ignore all recipe and recipe append + files in the meta-ti/recipes-misc/ + directory: + + BBMASK = "meta-ti/recipes-misc/" + + If you want to mask out multiple directories or recipes, + you can specify multiple regular expression fragments. + This next example masks out multiple directories and + individual recipes: + + BBMASK += "/meta-ti/recipes-misc/ meta-ti/recipes-ti/packagegroup/" + BBMASK += "/meta-oe/recipes-support/" + BBMASK += "/meta-foo/.*/openldap" + BBMASK += "opencv.*\.bbappend" + BBMASK += "lzma" + + + When specifying a directory name, use the trailing + slash character to ensure you match just that directory + name. + + + + + + BBPATH + + + Used by BitBake to locate class + (.bbclass) and configuration + (.conf) files. + This variable is analogous to the + PATH variable. + + + + If you run BitBake from a directory outside of the + build directory, + you must be sure to set + BBPATH to point to the + build directory. + Set the variable as you would any environment variable + and then run BitBake: + + $ BBPATH="build_directory" + $ export BBPATH + $ bitbake target + + + + + + BBSERVER + + + Points to the server that runs memory-resident BitBake. + The variable is only used when you employ memory-resident + BitBake. + + + + + BBTARGETS + + + Allows you to use a configuration file to add to the list + of command-line target recipes you want to build. + + + + + BBVERSIONS + + + Allows a single recipe to build multiple versions of a + project from a single recipe file. + You also able to specify conditional metadata + using the + OVERRIDES + mechanism for a single version or for an optionally named + range of versions. + + + + For more information on BBVERSIONS, + see the + "Variants - Class Extension Mechanism" + section. + + + + + BITBAKE_UI + + + Used to specify the UI module to use when running BitBake. + Using this variable is equivalent to using the + -u command-line option. + + You must set this variable in the external environment + in order for it to work. + + + + + + BUILDNAME + + + A name assigned to the build. + The name defaults to a datetime stamp of when the build was + started but can be defined by the metadata. + + + + + BZRDIR + + + The directory in which files checked out of a Bazaar + system are stored. + + + + + + + C + + CACHE + + + Specifies the directory BitBake uses to store a cache + of the metadata so it does not need to be parsed every + time BitBake is started. + + + + + CVSDIR + + + The directory in which files checked out under the + CVS system are stored. + + + + + + + D + + DEFAULT_PREFERENCE + + + Specifies a weak bias for recipe selection priority. + + + The most common usage of this is variable is to set + it to "-1" within a recipe for a development version of a + piece of software. + Using the variable in this way causes the stable version + of the recipe to build by default in the absence of + PREFERRED_VERSION + being used to build the development version. + + + The bias provided by DEFAULT_PREFERENCE + is weak and is overridden by + BBFILE_PRIORITY + if that variable is different between two layers + that contain different versions of the same recipe. + + + + + DEPENDS + + + Lists a recipe's build-time dependencies + (i.e. other recipe files). + + + + Consider this simple example for two recipes named "a" and + "b" that produce similarly named packages. + In this example, the DEPENDS + statement appears in the "a" recipe: + + DEPENDS = "b" + + Here, the dependency is such that the + do_configure task for recipe "a" + depends on the do_populate_sysroot + task of recipe "b". + This means anything that recipe "b" puts into sysroot + is available when recipe "a" is configuring itself. + + + + For information on runtime dependencies, see the + RDEPENDS + variable. + + + + + DESCRIPTION + + + A long description for the recipe. + + + + + DL_DIR + + + The central download directory used by the build process to + store downloads. + By default, DL_DIR gets files + suitable for mirroring for everything except Git + repositories. + If you want tarballs of Git repositories, use the + BB_GENERATE_MIRROR_TARBALLS + variable. + + + + + + + E + + EXCLUDE_FROM_WORLD + + + Directs BitBake to exclude a recipe from world builds (i.e. + bitbake world). + During world builds, BitBake locates, parses and builds all + recipes found in every layer exposed in the + bblayers.conf configuration file. + + + + To exclude a recipe from a world build using this variable, + set the variable to "1" in the recipe. + + + + Recipes added to EXCLUDE_FROM_WORLD + may still be built during a world build in order to satisfy + dependencies of other recipes. + Adding a recipe to EXCLUDE_FROM_WORLD + only ensures that the recipe is not explicitly added + to the list of build targets in a world build. + + + + + + + F + + FAKEROOT + + + Contains the command to use when running a shell script + in a fakeroot environment. + The FAKEROOT variable is obsolete + and has been replaced by the other + FAKEROOT* variables. + See these entries in the glossary for more information. + + + + + FAKEROOTBASEENV + + + Lists environment variables to set when executing + the command defined by + FAKEROOTCMD + that starts the bitbake-worker process + in the fakeroot environment. + + + + + FAKEROOTCMD + + + Contains the command that starts the bitbake-worker + process in the fakeroot environment. + + + + + FAKEROOTDIRS + + + Lists directories to create before running a task in + the fakeroot environment. + + + + + FAKEROOTENV + + + Lists environment variables to set when running a task + in the fakeroot environment. + For additional information on environment variables and + the fakeroot environment, see the + FAKEROOTBASEENV + variable. + + + + + FAKEROOTNOENV + + + Lists environment variables to set when running a task + that is not in the fakeroot environment. + For additional information on environment variables and + the fakeroot environment, see the + FAKEROOTENV + variable. + + + + + FETCHCMD + + + Defines the command the BitBake fetcher module + executes when running fetch operations. + You need to use an override suffix when you use the + variable (e.g. FETCHCMD_git + or FETCHCMD_svn). + + + + + FILE + + + Points at the current file. + BitBake sets this variable during the parsing process + to identify the file being parsed. + BitBake also sets this variable when a recipe is being + executed to identify the recipe file. + + + + + FILESDIR + + + Specifies directories BitBake uses when searching for + patches and files. + The "local" fetcher module uses these directories when + handling file:// URLs if the file + was not found using + FILESPATH. + + The FILESDIR variable is + deprecated and you should use + FILESPATH in all new code. + + + + + + FILESPATH + + + Specifies directories BitBake uses when searching for + patches and files. + The "local" fetcher module uses these directories when + handling file:// URLs. + The variable behaves like a shell PATH + environment variable. + The value is a colon-separated list of directories that + are searched left-to-right in order. + + + + + + + + G + + GITDIR + + + The directory in which a local copy of a Git repository + is stored when it is cloned. + + + + + + + + H + + HGDIR + + + The directory in which files checked out of a Mercurial + system are stored. + + + + + HOMEPAGE + + Website where more information about the software the recipe is building + can be found. + + + + + + I + + INHERIT + + + Causes the named class to be inherited at + this point during parsing. + The variable is only valid in configuration files. + + + + + + + + + L + + LAYERDEPENDS + + Lists the layers, separated by spaces, upon which this recipe depends. + Optionally, you can specify a specific layer version for a dependency + by adding it to the end of the layer name with a colon, (e.g. "anotherlayer:3" + to be compared against + LAYERVERSION_anotherlayer + in this case). + BitBake produces an error if any dependency is missing or + the version numbers do not match exactly (if specified). + + You use this variable in the conf/layer.conf file. + You must also use the specific layer name as a suffix + to the variable (e.g. LAYERDEPENDS_mylayer). + + + + LAYERDIR + + When used inside the layer.conf configuration + file, this variable provides the path of the current layer. + This variable is not available outside of layer.conf + and references are expanded immediately when parsing of the file completes. + + + + LAYERVERSION + + Optionally specifies the version of a layer as a single number. + You can use this variable within + LAYERDEPENDS + for another layer in order to depend on a specific version + of the layer. + + You use this variable in the conf/layer.conf file. + You must also use the specific layer name as a suffix + to the variable (e.g. LAYERDEPENDS_mylayer). + + + + LICENSE + + + The list of source licenses for the recipe. + + + + + + + M + + MIRRORS + + + Specifies additional paths from which BitBake gets source code. + When the build system searches for source code, it first + tries the local download directory. + If that location fails, the build system tries locations + defined by + PREMIRRORS, + the upstream source, and then locations specified by + MIRRORS in that order. + + + + + MULTI_PROVIDER_WHITELIST + + + Allows you to suppress BitBake warnings caused when + building two separate recipes that provide the same + output. + + + + Bitbake normally issues a warning when building two + different recipes where each provides the same output. + This scenario is usually something the user does not + want. + However, cases do exist where it makes sense, particularly + in the virtual/* namespace. + You can use this variable to suppress BitBake's warnings. + + + + To use the variable, list provider names (e.g. + recipe names, virtual/kernel, + and so forth). + + + + + + + + + O + + OVERRIDES + + + BitBake uses OVERRIDES to control + what variables are overridden after BitBake parses + recipes and configuration files. + + + + Following is a simple example that uses an overrides + list based on machine architectures: + + OVERRIDES = "arm:x86:mips:powerpc" + + You can find information on how to use + OVERRIDES in the + "Conditional Syntax (Overrides)" + section. + + + + + + P + + PACKAGES + + The list of packages the recipe creates. + + + + + PACKAGES_DYNAMIC + + + A promise that your recipe satisfies runtime dependencies + for optional modules that are found in other recipes. + PACKAGES_DYNAMIC + does not actually satisfy the dependencies, it only states that + they should be satisfied. + For example, if a hard, runtime dependency + (RDEPENDS) + of another package is satisfied during the build + through the PACKAGES_DYNAMIC + variable, but a package with the module name is never actually + produced, then the other package will be broken. + + + + + PE + + + The epoch of the recipe. + By default, this variable is unset. + The variable is used to make upgrades possible when the + versioning scheme changes in some backwards incompatible + way. + + + + + PERSISTENT_DIR + + + Specifies the directory BitBake uses to store data that + should be preserved between builds. + In particular, the data stored is the data that uses + BitBake's persistent data API and the data used by the + PR Server and PR Service. + + + + + PF + + + Specifies the recipe or package name and includes all version and revision + numbers (i.e. eglibc-2.13-r20+svnr15508/ and + bash-4.2-r1/). + + + + + PN + + The recipe name. + + + + PR + + The revision of the recipe. + + + + + PREFERRED_PROVIDER + + + Determines which recipe should be given preference when + multiple recipes provide the same item. + You should always suffix the variable with the name of the + provided item, and you should set it to the + PN + of the recipe to which you want to give precedence. + Some examples: + + PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto" + PREFERRED_PROVIDER_virtual/xserver = "xserver-xf86" + PREFERRED_PROVIDER_virtual/libgl ?= "mesa" + + + + + + PREFERRED_PROVIDERS + + + Determines which recipe should be given preference for + cases where multiple recipes provide the same item. + Functionally, + PREFERRED_PROVIDERS is identical to + PREFERRED_PROVIDER. + However, the PREFERRED_PROVIDERS + variable lets you define preferences for multiple + situations using the following form: + + PREFERRED_PROVIDERS = "xxx:yyy aaa:bbb ..." + + This form is a convenient replacement for the following: + + PREFERRED_PROVIDER_xxx = "yyy" + PREFERRED_PROVIDER_aaa = "bbb" + + + + + + PREFERRED_VERSION + + + If there are multiple versions of recipes available, this + variable determines which recipe should be given preference. + You must always suffix the variable with the + PN + you want to select, and you should set + PV + accordingly for precedence. + You can use the "%" character as a + wildcard to match any number of characters, which can be + useful when specifying versions that contain long revision + numbers that could potentially change. + Here are two examples: + + PREFERRED_VERSION_python = "2.7.3" + PREFERRED_VERSION_linux-yocto = "3.10%" + + + + + + PREMIRRORS + + + Specifies additional paths from which BitBake gets source code. + When the build system searches for source code, it first + tries the local download directory. + If that location fails, the build system tries locations + defined by PREMIRRORS, the upstream + source, and then locations specified by + MIRRORS + in that order. + + + + Typically, you would add a specific server for the + build system to attempt before any others by adding + something like the following to your configuration: + + PREMIRRORS_prepend = "\ + git://.*/.* http://www.yoctoproject.org/sources/ \n \ + ftp://.*/.* http://www.yoctoproject.org/sources/ \n \ + http://.*/.* http://www.yoctoproject.org/sources/ \n \ + https://.*/.* http://www.yoctoproject.org/sources/ \n" + + These changes cause the build system to intercept + Git, FTP, HTTP, and HTTPS requests and direct them to + the http:// sources mirror. + You can use file:// URLs to point + to local directories or network shares as well. + + + + + PROVIDES + + + A list of aliases by which a particular recipe can be + known. + By default, a recipe's own + PN + is implicitly already in its PROVIDES + list. + If a recipe uses PROVIDES, the + additional aliases are synonyms for the recipe and can + be useful satisfying dependencies of other recipes during + the build as specified by + DEPENDS. + + + + Consider the following example + PROVIDES statement from a recipe + file libav_0.8.11.bb: + + PROVIDES += "libpostproc" + + The PROVIDES statement results in + the "libav" recipe also being known as "libpostproc". + + + + + PRSERV_HOST + + + The network based + PR + service host and port. + + + + Following is an example of how the PRSERV_HOST variable is + set: + + PRSERV_HOST = "localhost:0" + + You must set the variable if you want to automatically + start a local PR service. + You can set PRSERV_HOST to other + values to use a remote PR service. + + + + + PV + + The version of the recipe. + + + + + + + + + R + + RDEPENDS + + + Lists a package's runtime dependencies (i.e. other packages) + that must be installed in order for the built package to run + correctly. + If a package in this list cannot be found during the build, + you will get a build error. + + + + Because the RDEPENDS variable applies + to packages being built, you should always use the variable + in a form with an attached package name. + For example, suppose you are building a development package + that depends on the perl package. + In this case, you would use the following + RDEPENDS statement: + + RDEPENDS_${PN}-dev += "perl" + + In the example, the development package depends on + the perl package. + Thus, the RDEPENDS variable has the + ${PN}-dev package name as part of the + variable. + + + + BitBake supports specifying versioned dependencies. + Although the syntax varies depending on the packaging + format, BitBake hides these differences from you. + Here is the general syntax to specify versions with + the RDEPENDS variable: + + RDEPENDS_${PN} = "package (operator version)" + + For operator, you can specify the + following: + + = + < + > + <= + >= + + For example, the following sets up a dependency on version + 1.2 or greater of the package foo: + + RDEPENDS_${PN} = "foo (>= 1.2)" + + + + + For information on build-time dependencies, see the + DEPENDS + variable. + + + + + RPROVIDES + + + A list of package name aliases that a package also provides. + These aliases are useful for satisfying runtime dependencies + of other packages both during the build and on the target + (as specified by + RDEPENDS). + + + As with all package-controlling variables, you must always + use the variable in conjunction with a package name override. + Here is an example: + + RPROVIDES_${PN} = "widget-abi-2" + + + + + + RRECOMMENDS + + + A list of packages that extends the usability of a package + being built. + The package being built does not depend on this list of + packages in order to successfully build, but needs them for + the extended usability. + To specify runtime dependencies for packages, see the + RDEPENDS + variable. + + + + BitBake supports specifying versioned recommends. + Although the syntax varies depending on the packaging + format, BitBake hides these differences from you. + Here is the general syntax to specify versions with + the RRECOMMENDS variable: + + RRECOMMENDS_${PN} = "package (operator version)" + + For operator, you can specify the + following: + + = + < + > + <= + >= + + For example, the following sets up a recommend on version + 1.2 or greater of the package foo: + + RRECOMMENDS_${PN} = "foo (>= 1.2)" + + + + + + + + S + + SECTION + + The section in which packages should be categorized. + + + + SRC_URI + + + The list of source files - local or remote. + This variable tells BitBake which bits + to pull for the build and how to pull them. + For example, if the recipe or append file needs to + fetch a single tarball from the Internet, the recipe or + append file uses a SRC_URI + entry that specifies that tarball. + On the other hand, if the recipe or append file needs to + fetch a tarball and include a custom file, the recipe or + append file needs an SRC_URI variable + that specifies all those sources. + The following list explains the available URI protocols: + + file:// - + Fetches files, which are usually files shipped with + the metadata, + from the local machine. + The path is relative to the + FILESPATH + variable. + bzr:// - Fetches files from a + Bazaar revision control repository. + git:// - Fetches files from a + Git revision control repository. + osc:// - Fetches files from + an OSC (OpenSUSE Build service) revision control repository. + repo:// - Fetches files from + a repo (Git) repository. + http:// - Fetches files from + the Internet using HTTP. + https:// - Fetches files + from the Internet using HTTPS. + ftp:// - Fetches files + from the Internet using FTP. + cvs:// - Fetches files from + a CVS revision control repository. + hg:// - Fetches files from + a Mercurial (hg) revision control repository. + p4:// - Fetches files from + a Perforce (p4) revision control repository. + ssh:// - Fetches files from + a secure shell. + svn:// - Fetches files from + a Subversion (svn) revision control repository. + + + Here are some additional options worth mentioning: + + unpack - Controls + whether or not to unpack the file if it is an archive. + The default action is to unpack the file. + subdir - Places the file + (or extracts its contents) into the specified + subdirectory. + This option is useful for unusual tarballs or other archives that + do not have their files already in a subdirectory within the archive. + + name - Specifies a + name to be used for association with SRC_URI checksums + when you have more than one file specified in SRC_URI. + + downloadfilename - Specifies + the filename used when storing the downloaded file. + + + + + + SRCDATE + + + The date of the source code used to build the package. + This variable applies only if the source was fetched from a Source Code Manager (SCM). + + + + + SRCREV + + + The revision of the source code used to build the package. + This variable applies only when using Subversion, Git, Mercurial and Bazaar. + If you want to build a fixed revision and you want + to avoid performing a query on the remote repository every time + BitBake parses your recipe, you should specify a SRCREV that is a + full revision identifier and not just a tag. + + + + + SRCREV_FORMAT + + + Helps construct valid + SRCREV + values when multiple source controlled URLs are used in + SRC_URI. + + + + The system needs help constructing these values under these + circumstances. + Each component in the SRC_URI + is assigned a name and these are referenced + in the SRCREV_FORMAT variable. + Consider an example with URLs named "machine" and "meta". + In this case, SRCREV_FORMAT could look + like "machine_meta" and those names would have the SCM + versions substituted into each position. + Only one AUTOINC placeholder is added + and if needed. + And, this placeholder is placed at the start of the + returned string. + + + + + STAMP + + + Specifies the base path used to create recipe stamp files. + The path to an actual stamp file is constructed by evaluating this + string and then appending additional information. + + + + + STAMPCLEAN + + + Specifies the base path used to create recipe stamp files. + Unlike the + STAMP + variable, STAMPCLEAN can contain + wildcards to match the range of files a clean operation + should remove. + BitBake uses a clean operation to remove any other stamps + it should be removing when creating a new stamp. + + + + + SUMMARY + + + A short summary for the recipe, which is 72 characters or less. + + + + + SVNDIR + + + The directory in which files checked out of a Subversion + system are stored. + + + + + + + T + + T + + Points to a directory were BitBake places + temporary files, which consist mostly of task logs and + scripts, when building a particular recipe. + + + + + TOPDIR + + + Points to the build directory. + BitBake automatically sets this variable. + + + + + + + + + + + + diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-style.css b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-style.css new file mode 100644 index 000000000..65da2a4e3 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-style.css @@ -0,0 +1,984 @@ +/* + Generic XHTML / DocBook XHTML CSS Stylesheet. + + Browser wrangling and typographic design by + Oyvind Kolas / pippin@gimp.org + + Customised for Poky by + Matthew Allum / mallum@o-hand.com + + Thanks to: + Liam R. E. Quin + William Skaggs + Jakub Steiner + + Structure + --------- + + The stylesheet is divided into the following sections: + + Positioning + Margins, paddings, width, font-size, clearing. + Decorations + Borders, style + Colors + Colors + Graphics + Graphical backgrounds + Nasty IE tweaks + Workarounds needed to make it work in internet explorer, + currently makes the stylesheet non validating, but up until + this point it is validating. + Mozilla extensions + Transparency for footer + Rounded corners on boxes + +*/ + + + /*************** / + / Positioning / +/ ***************/ + +body { + font-family: Verdana, Sans, sans-serif; + + min-width: 640px; + width: 80%; + margin: 0em auto; + padding: 2em 5em 5em 5em; + color: #333; +} + +h1,h2,h3,h4,h5,h6,h7 { + font-family: Arial, Sans; + color: #00557D; + clear: both; +} + +h1 { + font-size: 2em; + text-align: left; + padding: 0em 0em 0em 0em; + margin: 2em 0em 0em 0em; +} + +h2.subtitle { + margin: 0.10em 0em 3.0em 0em; + padding: 0em 0em 0em 0em; + font-size: 1.8em; + padding-left: 20%; + font-weight: normal; + font-style: italic; +} + +h2 { + margin: 2em 0em 0.66em 0em; + padding: 0.5em 0em 0em 0em; + font-size: 1.5em; + font-weight: bold; +} + +h3.subtitle { + margin: 0em 0em 1em 0em; + padding: 0em 0em 0em 0em; + font-size: 142.14%; + text-align: right; +} + +h3 { + margin: 1em 0em 0.5em 0em; + padding: 1em 0em 0em 0em; + font-size: 140%; + font-weight: bold; +} + +h4 { + margin: 1em 0em 0.5em 0em; + padding: 1em 0em 0em 0em; + font-size: 120%; + font-weight: bold; +} + +h5 { + margin: 1em 0em 0.5em 0em; + padding: 1em 0em 0em 0em; + font-size: 110%; + font-weight: bold; +} + +h6 { + margin: 1em 0em 0em 0em; + padding: 1em 0em 0em 0em; + font-size: 110%; + font-weight: bold; +} + +.authorgroup { + background-color: transparent; + background-repeat: no-repeat; + padding-top: 256px; + background-image: url("figures/bitbake-title.png"); + background-position: left top; + margin-top: -256px; + padding-right: 50px; + margin-left: 0px; + text-align: right; + width: 740px; +} + +h3.author { + margin: 0em 0me 0em 0em; + padding: 0em 0em 0em 0em; + font-weight: normal; + font-size: 100%; + color: #333; + clear: both; +} + +.author tt.email { + font-size: 66%; +} + +.titlepage hr { + width: 0em; + clear: both; +} + +.revhistory { + padding-top: 2em; + clear: both; +} + +.toc, +.list-of-tables, +.list-of-examples, +.list-of-figures { + padding: 1.33em 0em 2.5em 0em; + color: #00557D; +} + +.toc p, +.list-of-tables p, +.list-of-figures p, +.list-of-examples p { + padding: 0em 0em 0em 0em; + padding: 0em 0em 0.3em; + margin: 1.5em 0em 0em 0em; +} + +.toc p b, +.list-of-tables p b, +.list-of-figures p b, +.list-of-examples p b{ + font-size: 100.0%; + font-weight: bold; +} + +.toc dl, +.list-of-tables dl, +.list-of-figures dl, +.list-of-examples dl { + margin: 0em 0em 0.5em 0em; + padding: 0em 0em 0em 0em; +} + +.toc dt { + margin: 0em 0em 0em 0em; + padding: 0em 0em 0em 0em; +} + +.toc dd { + margin: 0em 0em 0em 2.6em; + padding: 0em 0em 0em 0em; +} + +div.glossary dl, +div.variablelist dl { +} + +.glossary dl dt, +.variablelist dl dt, +.variablelist dl dt span.term { + font-weight: normal; + width: 20em; + text-align: right; +} + +.variablelist dl dt { + margin-top: 0.5em; +} + +.glossary dl dd, +.variablelist dl dd { + margin-top: -1em; + margin-left: 25.5em; +} + +.glossary dd p, +.variablelist dd p { + margin-top: 0em; + margin-bottom: 1em; +} + + +div.calloutlist table td { + padding: 0em 0em 0em 0em; + margin: 0em 0em 0em 0em; +} + +div.calloutlist table td p { + margin-top: 0em; + margin-bottom: 1em; +} + +div p.copyright { + text-align: left; +} + +div.legalnotice p.legalnotice-title { + margin-bottom: 0em; +} + +p { + line-height: 1.5em; + margin-top: 0em; + +} + +dl { + padding-top: 0em; +} + +hr { + border: solid 1px; +} + + +.mediaobject, +.mediaobjectco { + text-align: center; +} + +img { + border: none; +} + +ul { + padding: 0em 0em 0em 1.5em; +} + +ul li { + padding: 0em 0em 0em 0em; +} + +ul li p { + text-align: left; +} + +table { + width :100%; +} + +th { + padding: 0.25em; + text-align: left; + font-weight: normal; + vertical-align: top; +} + +td { + padding: 0.25em; + vertical-align: top; +} + +p a[id] { + margin: 0px; + padding: 0px; + display: inline; + background-image: none; +} + +a { + text-decoration: underline; + color: #444; +} + +pre { + overflow: auto; +} + +a:hover { + text-decoration: underline; + /*font-weight: bold;*/ +} + +/* This style defines how the permalink character + appears by itself and when hovered over with + the mouse. */ + +[alt='Permalink'] { color: #eee; } +[alt='Permalink']:hover { color: black; } + + +div.informalfigure, +div.informalexample, +div.informaltable, +div.figure, +div.table, +div.example { + margin: 1em 0em; + padding: 1em; + page-break-inside: avoid; +} + + +div.informalfigure p.title b, +div.informalexample p.title b, +div.informaltable p.title b, +div.figure p.title b, +div.example p.title b, +div.table p.title b{ + padding-top: 0em; + margin-top: 0em; + font-size: 100%; + font-weight: normal; +} + +.mediaobject .caption, +.mediaobject .caption p { + text-align: center; + font-size: 80%; + padding-top: 0.5em; + padding-bottom: 0.5em; +} + +.epigraph { + padding-left: 55%; + margin-bottom: 1em; +} + +.epigraph p { + text-align: left; +} + +.epigraph .quote { + font-style: italic; +} +.epigraph .attribution { + font-style: normal; + text-align: right; +} + +span.application { + font-style: italic; +} + +.programlisting { + font-family: monospace; + font-size: 80%; + white-space: pre; + margin: 1.33em 0em; + padding: 1.33em; +} + +.tip, +.warning, +.caution, +.note { + margin-top: 1em; + margin-bottom: 1em; + +} + +/* force full width of table within div */ +.tip table, +.warning table, +.caution table, +.note table { + border: none; + width: 100%; +} + + +.tip table th, +.warning table th, +.caution table th, +.note table th { + padding: 0.8em 0.0em 0.0em 0.0em; + margin : 0em 0em 0em 0em; +} + +.tip p, +.warning p, +.caution p, +.note p { + margin-top: 0.5em; + margin-bottom: 0.5em; + padding-right: 1em; + text-align: left; +} + +.acronym { + text-transform: uppercase; +} + +b.keycap, +.keycap { + padding: 0.09em 0.3em; + margin: 0em; +} + +.itemizedlist li { + clear: none; +} + +.filename { + font-size: medium; + font-family: Courier, monospace; +} + + +div.navheader, div.heading{ + position: absolute; + left: 0em; + top: 0em; + width: 100%; + background-color: #cdf; + width: 100%; +} + +div.navfooter, div.footing{ + position: fixed; + left: 0em; + bottom: 0em; + background-color: #eee; + width: 100%; +} + + +div.navheader td, +div.navfooter td { + font-size: 66%; +} + +div.navheader table th { + /*font-family: Georgia, Times, serif;*/ + /*font-size: x-large;*/ + font-size: 80%; +} + +div.navheader table { + border-left: 0em; + border-right: 0em; + border-top: 0em; + width: 100%; +} + +div.navfooter table { + border-left: 0em; + border-right: 0em; + border-bottom: 0em; + width: 100%; +} + +div.navheader table td a, +div.navfooter table td a { + color: #777; + text-decoration: none; +} + +/* normal text in the footer */ +div.navfooter table td { + color: black; +} + +div.navheader table td a:visited, +div.navfooter table td a:visited { + color: #444; +} + + +/* links in header and footer */ +div.navheader table td a:hover, +div.navfooter table td a:hover { + text-decoration: underline; + background-color: transparent; + color: #33a; +} + +div.navheader hr, +div.navfooter hr { + display: none; +} + + +.qandaset tr.question td p { + margin: 0em 0em 1em 0em; + padding: 0em 0em 0em 0em; +} + +.qandaset tr.answer td p { + margin: 0em 0em 1em 0em; + padding: 0em 0em 0em 0em; +} +.answer td { + padding-bottom: 1.5em; +} + +.emphasis { + font-weight: bold; +} + + + /************* / + / decorations / +/ *************/ + +.titlepage { +} + +.part .title { +} + +.subtitle { + border: none; +} + +/* +h1 { + border: none; +} + +h2 { + border-top: solid 0.2em; + border-bottom: solid 0.06em; +} + +h3 { + border-top: 0em; + border-bottom: solid 0.06em; +} + +h4 { + border: 0em; + border-bottom: solid 0.06em; +} + +h5 { + border: 0em; +} +*/ + +.programlisting { + border: solid 1px; +} + +div.figure, +div.table, +div.informalfigure, +div.informaltable, +div.informalexample, +div.example { + border: 1px solid; +} + + + +.tip, +.warning, +.caution, +.note { + border: 1px solid; +} + +.tip table th, +.warning table th, +.caution table th, +.note table th { + border-bottom: 1px solid; +} + +.question td { + border-top: 1px solid black; +} + +.answer { +} + + +b.keycap, +.keycap { + border: 1px solid; +} + + +div.navheader, div.heading{ + border-bottom: 1px solid; +} + + +div.navfooter, div.footing{ + border-top: 1px solid; +} + + /********* / + / colors / +/ *********/ + +body { + color: #333; + background: white; +} + +a { + background: transparent; +} + +a:hover { + background-color: #dedede; +} + + +h1, +h2, +h3, +h4, +h5, +h6, +h7, +h8 { + background-color: transparent; +} + +hr { + border-color: #aaa; +} + + +.tip, .warning, .caution, .note { + border-color: #fff; +} + + +.tip table th, +.warning table th, +.caution table th, +.note table th { + border-bottom-color: #fff; +} + + +.warning { + background-color: #f0f0f2; +} + +.caution { + background-color: #f0f0f2; +} + +.tip { + background-color: #f0f0f2; +} + +.note { + background-color: #f0f0f2; +} + +.glossary dl dt, +.variablelist dl dt, +.variablelist dl dt span.term { + color: #044; +} + +div.figure, +div.table, +div.example, +div.informalfigure, +div.informaltable, +div.informalexample { + border-color: #aaa; +} + +pre.programlisting { + color: black; + background-color: #fff; + border-color: #aaa; + border-width: 2px; +} + +.guimenu, +.guilabel, +.guimenuitem { + background-color: #eee; +} + + +b.keycap, +.keycap { + background-color: #eee; + border-color: #999; +} + + +div.navheader { + border-color: black; +} + + +div.navfooter { + border-color: black; +} + + + /*********** / + / graphics / +/ ***********/ + +/* +body { + background-image: url("images/body_bg.jpg"); + background-attachment: fixed; +} + +.navheader, +.note, +.tip { + background-image: url("images/note_bg.jpg"); + background-attachment: fixed; +} + +.warning, +.caution { + background-image: url("images/warning_bg.jpg"); + background-attachment: fixed; +} + +.figure, +.informalfigure, +.example, +.informalexample, +.table, +.informaltable { + background-image: url("images/figure_bg.jpg"); + background-attachment: fixed; +} + +*/ +h1, +h2, +h3, +h4, +h5, +h6, +h7{ +} + +/* +Example of how to stick an image as part of the title. + +div.article .titlepage .title +{ + background-image: url("figures/white-on-black.png"); + background-position: center; + background-repeat: repeat-x; +} +*/ + +div.preface .titlepage .title, +div.colophon .title, +div.chapter .titlepage .title, +div.article .titlepage .title +{ +} + +div.section div.section .titlepage .title, +div.sect2 .titlepage .title { + background: none; +} + + +h1.title { + background-color: transparent; + background-repeat: no-repeat; + height: 256px; + text-indent: -9000px; + overflow:hidden; +} + +h2.subtitle { + background-color: transparent; + text-indent: -9000px; + overflow:hidden; + width: 0px; + display: none; +} + + /*************************************** / + / pippin.gimp.org specific alterations / +/ ***************************************/ + +/* +div.heading, div.navheader { + color: #777; + font-size: 80%; + padding: 0; + margin: 0; + text-align: left; + position: absolute; + top: 0px; + left: 0px; + width: 100%; + height: 50px; + background: url('/gfx/heading_bg.png') transparent; + background-repeat: repeat-x; + background-attachment: fixed; + border: none; +} + +div.heading a { + color: #444; +} + +div.footing, div.navfooter { + border: none; + color: #ddd; + font-size: 80%; + text-align:right; + + width: 100%; + padding-top: 10px; + position: absolute; + bottom: 0px; + left: 0px; + + background: url('/gfx/footing_bg.png') transparent; +} +*/ + + + + /****************** / + / nasty ie tweaks / +/ ******************/ + +/* +div.heading, div.navheader { + width:expression(document.body.clientWidth + "px"); +} + +div.footing, div.navfooter { + width:expression(document.body.clientWidth + "px"); + margin-left:expression("-5em"); +} +body { + padding:expression("4em 5em 0em 5em"); +} +*/ + + /**************************************** / + / mozilla vendor specific css extensions / +/ ****************************************/ +/* +div.navfooter, div.footing{ + -moz-opacity: 0.8em; +} + +div.figure, +div.table, +div.informalfigure, +div.informaltable, +div.informalexample, +div.example, +.tip, +.warning, +.caution, +.note { + -moz-border-radius: 0.5em; +} + +b.keycap, +.keycap { + -moz-border-radius: 0.3em; +} +*/ + +table tr td table tr td { + display: none; +} + + +hr { + display: none; +} + +table { + border: 0em; +} + + .photo { + float: right; + margin-left: 1.5em; + margin-bottom: 1.5em; + margin-top: 0em; + max-width: 17em; + border: 1px solid gray; + padding: 3px; + background: white; +} + .seperator { + padding-top: 2em; + clear: both; + } + + #validators { + margin-top: 5em; + text-align: right; + color: #777; + } + @media print { + body { + font-size: 8pt; + } + .noprint { + display: none; + } + } + + +.tip, +.note { + background: #f0f0f2; + color: #333; + padding: 20px; + margin: 20px; +} + +.tip h3, +.note h3 { + padding: 0em; + margin: 0em; + font-size: 2em; + font-weight: bold; + color: #333; +} + +.tip a, +.note a { + color: #333; + text-decoration: underline; +} + +.footnote { + font-size: small; + color: #333; +} + +/* Changes the announcement text */ +.tip h3, +.warning h3, +.caution h3, +.note h3 { + font-size:large; + color: #00557D; +} diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml new file mode 100644 index 000000000..4aef4e72f --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + BitBake User Manual + + + + + Richard Purdie, Chris Larson, and Phil Blundell + + BitBake Community + + bitbake-devel@lists.openembedded.org + + + + + + + 2004-2016 + Richard Purdie + Chris Larson + and Phil Blundell + + + + + This work is licensed under the Creative Commons Attribution License. + To view a copy of this license, visit + http://creativecommons.org/licenses/by/2.5/ + or send a letter to Creative Commons, 444 Castro Street, + Suite 900, Mountain View, California 94041, USA. + + + + + + + + + + + + + + + + + diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/figures/bitbake-title.png b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/figures/bitbake-title.png new file mode 100644 index 000000000..cb290154d Binary files /dev/null and b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/figures/bitbake-title.png differ diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/html.css b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/html.css new file mode 100644 index 000000000..6eedfd318 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/html.css @@ -0,0 +1,281 @@ +/* Feuille de style DocBook du projet Traduc.org */ +/* DocBook CSS stylesheet of the Traduc.org project */ + +/* (c) Jean-Philippe Guérard - 14 août 2004 */ +/* (c) Jean-Philippe Guérard - 14 August 2004 */ + +/* Cette feuille de style est libre, vous pouvez la */ +/* redistribuer et la modifier selon les termes de la Licence */ +/* Art Libre. Vous trouverez un exemplaire de cette Licence sur */ +/* http://tigreraye.org/Petit-guide-du-traducteur.html#licence-art-libre */ + +/* This work of art is free, you can redistribute it and/or */ +/* modify it according to terms of the Free Art license. You */ +/* will find a specimen of this license on the Copyleft */ +/* Attitude web site: http://artlibre.org as well as on other */ +/* sites. */ +/* Please note that the French version of this licence as shown */ +/* on http://tigreraye.org/Petit-guide-du-traducteur.html#licence-art-libre */ +/* is only official licence of this document. The English */ +/* is only provided to help you understand this licence. */ + +/* La dernière version de cette feuille de style est toujours */ +/* disponible sur : http://tigreraye.org/style.css */ +/* Elle est également disponible sur : */ +/* http://www.traduc.org/docs/HOWTO/lecture/style.css */ + +/* The latest version of this stylesheet is available from: */ +/* http://tigreraye.org/style.css */ +/* It is also available on: */ +/* http://www.traduc.org/docs/HOWTO/lecture/style.css */ + +/* N'hésitez pas à envoyer vos commentaires et corrections à */ +/* Jean-Philippe Guérard */ + +/* Please send feedback and bug reports to */ +/* Jean-Philippe Guérard */ + +/* $Id: style.css,v 1.14 2004/09/10 20:12:09 fevrier Exp fevrier $ */ + +/* Présentation générale du document */ +/* Overall document presentation */ + +body { + /* + font-family: Apolline, "URW Palladio L", Garamond, jGaramond, + "Bitstream Cyberbit", "Palatino Linotype", serif; + */ + margin: 7%; + background-color: white; +} + +/* Taille du texte */ +/* Text size */ + +* { font-size: 100%; } + +/* Gestion des textes mis en relief imbriqués */ +/* Embedded emphasis */ + +em { font-style: italic; } +em em { font-style: normal; } +em em em { font-style: italic; } + +/* Titres */ +/* Titles */ + +h1 { font-size: 200%; font-weight: 900; } +h2 { font-size: 160%; font-weight: 900; } +h3 { font-size: 130%; font-weight: bold; } +h4 { font-size: 115%; font-weight: bold; } +h5 { font-size: 108%; font-weight: bold; } +h6 { font-weight: bold; } + +/* Nom de famille en petites majuscules (uniquement en français) */ +/* Last names in small caps (for French only) */ + +*[class~="surname"]:lang(fr) { font-variant: small-caps; } + +/* Blocs de citation */ +/* Quotation blocs */ + +div[class~="blockquote"] { + border: solid 2px #AAA; + padding: 5px; + margin: 5px; +} + +div[class~="blockquote"] > table { + border: none; +} + +/* Blocs litéraux : fond gris clair */ +/* Literal blocs: light gray background */ + +*[class~="literallayout"] { + background: #f0f0f0; + padding: 5px; + margin: 5px; +} + +/* Programmes et captures texte : fond bleu clair */ +/* Listing and text screen snapshots: light blue background */ + +*[class~="programlisting"], *[class~="screen"] { + background: #f0f0ff; + padding: 5px; + margin: 5px; +} + +/* Les textes à remplacer sont surlignés en vert pâle */ +/* Replaceable text in highlighted in pale green */ + +*[class~="replaceable"] { + background-color: #98fb98; + font-style: normal; } + +/* Tables : fonds gris clair & bords simples */ +/* Tables: light gray background and solid borders */ + +*[class~="table"] *[class~="title"] { width:100%; border: 0px; } + +table { + border: 1px solid #aaa; + border-collapse: collapse; + padding: 2px; + margin: 5px; +} + +/* Listes simples en style table */ +/* Simples lists in table presentation */ + +table[class~="simplelist"] { + background-color: #F0F0F0; + margin: 5px; + border: solid 1px #AAA; +} + +table[class~="simplelist"] td { + border: solid 1px #AAA; +} + +/* Les tables */ +/* Tables */ + +*[class~="table"] table { + background-color: #F0F0F0; + border: solid 1px #AAA; +} +*[class~="informaltable"] table { background-color: #F0F0F0; } + +th,td { + vertical-align: baseline; + text-align: left; + padding: 0.1em 0.3em; + empty-cells: show; +} + +/* Alignement des colonnes */ +/* Colunms alignment */ + +td[align=center] , th[align=center] { text-align: center; } +td[align=right] , th[align=right] { text-align: right; } +td[align=left] , th[align=left] { text-align: left; } +td[align=justify] , th[align=justify] { text-align: justify; } + +/* Pas de marge autour des images */ +/* No inside margins for images */ + +img { border: 0; } + +/* Les liens ne sont pas soulignés */ +/* No underlines for links */ + +:link , :visited , :active { text-decoration: none; } + +/* Prudence : cadre jaune et fond jaune clair */ +/* Caution: yellow border and light yellow background */ + +*[class~="caution"] { + border: solid 2px yellow; + background-color: #ffffe0; + padding: 1em 6px 1em ; + margin: 5px; +} + +*[class~="caution"] th { + vertical-align: middle +} + +*[class~="caution"] table { + background-color: #ffffe0; + border: none; +} + +/* Note importante : cadre jaune et fond jaune clair */ +/* Important: yellow border and light yellow background */ + +*[class~="important"] { + border: solid 2px yellow; + background-color: #ffffe0; + padding: 1em 6px 1em; + margin: 5px; +} + +*[class~="important"] th { + vertical-align: middle +} + +*[class~="important"] table { + background-color: #ffffe0; + border: none; +} + +/* Mise en évidence : texte légèrement plus grand */ +/* Highlights: slightly larger texts */ + +*[class~="highlights"] { + font-size: 110%; +} + +/* Note : cadre bleu et fond bleu clair */ +/* Notes: blue border and light blue background */ + +*[class~="note"] { + border: solid 2px #7099C5; + background-color: #f0f0ff; + padding: 1em 6px 1em ; + margin: 5px; +} + +*[class~="note"] th { + vertical-align: middle +} + +*[class~="note"] table { + background-color: #f0f0ff; + border: none; +} + +/* Astuce : cadre vert et fond vert clair */ +/* Tip: green border and light green background */ + +*[class~="tip"] { + border: solid 2px #00ff00; + background-color: #f0ffff; + padding: 1em 6px 1em ; + margin: 5px; +} + +*[class~="tip"] th { + vertical-align: middle; +} + +*[class~="tip"] table { + background-color: #f0ffff; + border: none; +} + +/* Avertissement : cadre rouge et fond rouge clair */ +/* Warning: red border and light red background */ + +*[class~="warning"] { + border: solid 2px #ff0000; + background-color: #fff0f0; + padding: 1em 6px 1em ; + margin: 5px; +} + +*[class~="warning"] th { + vertical-align: middle; +} + + +*[class~="warning"] table { + background-color: #fff0f0; + border: none; +} + +/* Fin */ +/* The End */ + diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake.1 b/import-layers/yocto-poky/bitbake/doc/bitbake.1 new file mode 100644 index 000000000..a6c8d9727 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/bitbake.1 @@ -0,0 +1,142 @@ +.\" Hey, EMACS: -*- nroff -*- +.\" First parameter, NAME, should be all caps +.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection +.\" other parameters are allowed: see man(7), man(1) +.TH BITBAKE 1 "November 19, 2006" +.\" Please adjust this date whenever revising the manpage. +.\" +.\" Some roff macros, for reference: +.\" .nh disable hyphenation +.\" .hy enable hyphenation +.\" .ad l left justify +.\" .ad b justify to both left and right margins +.\" .nf disable filling +.\" .fi enable filling +.\" .br insert line break +.\" .sp insert n+1 empty lines +.\" for manpage-specific macros, see man(7) +.SH NAME +BitBake \- simple tool for the execution of tasks +.SH SYNOPSIS +.B bitbake +.RI [ options ] " packagenames" +.br +.SH DESCRIPTION +This manual page documents briefly the +.B bitbake +command. +.PP +.\" TeX users may be more comfortable with the \fB\fP and +.\" \fI\fP escape sequences to invode bold face and italics, +.\" respectively. +\fBbitbake\fP is a program that executes the specified task (default is 'build') +for a given set of BitBake files. +.br +It expects that BBFILES is defined, which is a space separated list of files to +be executed. BBFILES does support wildcards. +.br +Default BBFILES are the .bb files in the current directory. +.SH OPTIONS +This program follow the usual GNU command line syntax, with long +options starting with two dashes (`-'). +.TP +.B \-h, \-\-help +Show summary of options. +.TP +.B \-\-version +Show version of program. +.TP +.B \-bBUILDFILE, \-\-buildfile=BUILDFILE +execute the task against this .bb file, rather than a package from BBFILES. +.TP +.B \-k, \-\-continue +continue as much as possible after an error. While the target that failed, and +those that depend on it, cannot be remade, the other dependencies of these +targets can be processed all the same. +.TP +.B \-a, \-\-tryaltconfigs +continue with builds by trying to use alternative providers where possible. +.TP +.B \-f, \-\-force +force run of specified cmd, regardless of stamp status +.TP +.B \-i, \-\-interactive +drop into the interactive mode also called the BitBake shell. +.TP +.B \-cCMD, \-\-cmd=CMD +Specify task to execute. Note that this only executes the specified task for +the providee and the packages it depends on, i.e. 'compile' does not implicitly +call stage for the dependencies (IOW: use only if you know what you are doing). +Depending on the base.bbclass a listtasks task is defined and will show +available tasks. +.TP +.B \-rFILE, \-\-read=FILE +read the specified file before bitbake.conf +.TP +.B \-v, \-\-verbose +output more chit-chat to the terminal +.TP +.B \-D, \-\-debug +Increase the debug level. You can specify this more than once. +.TP +.B \-n, \-\-dry-run +don't execute, just go through the motions +.TP +.B \-p, \-\-parse-only +quit after parsing the BB files (developers only) +.TP +.B \-s, \-\-show-versions +show current and preferred versions of all packages +.TP +.B \-e, \-\-environment +show the global or per-recipe environment (this is what used to be bbread) +.TP +.B \-g, \-\-graphviz +emit the dependency trees of the specified packages in the dot syntax +.TP +.B \-IIGNORED\_DOT\_DEPS, \-\-ignore-deps=IGNORED_DOT_DEPS +Stop processing at the given list of dependencies when generating dependency +graphs. This can help to make the graph more appealing +.TP +.B \-lDEBUG_DOMAINS, \-\-log-domains=DEBUG_DOMAINS +Show debug logging for the specified logging domains +.TP +.B \-P, \-\-profile +profile the command and print a report +.TP +.B \-uUI, \-\-ui=UI +User interface to use. Currently, hob, depexp, goggle or ncurses can be specified as UI. +.TP +.B \-tSERVERTYPE, \-\-servertype=SERVERTYPE +Choose which server to use, none, process or xmlrpc. +.TP +.B \-\-revisions-changed +Set the exit code depending on whether upstream floating revisions have changed or not. +.TP +.B \-\-server-only +Run bitbake without UI, the frontend can connect with bitbake server itself. +.TP +.B \-BBIND, \-\-bind=BIND +The name/address for the bitbake server to bind to. +.TP +.B \-\-no\-setscene +Do not run any setscene tasks, forces builds. + +.SH ENVIRONMENT VARIABLES +bitbake uses the following environment variables to control its +operation: +.TP +.B BITBAKE_UI +The bitbake user interface; overridden by the \fB-u\fP commandline option. + +.SH AUTHORS +BitBake was written by +Phil Blundell, +Holger Freyther, +Chris Larson, +Mickey Lauer, +Richard Purdie, +Holger Schurig +.PP +This manual page was written by Marcin Juszkiewicz +for the Debian project (but may be used by others). diff --git a/import-layers/yocto-poky/bitbake/doc/poky.ent b/import-layers/yocto-poky/bitbake/doc/poky.ent new file mode 100644 index 000000000..c032e1418 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/poky.ent @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/import-layers/yocto-poky/bitbake/doc/template/Vera.ttf b/import-layers/yocto-poky/bitbake/doc/template/Vera.ttf new file mode 100644 index 000000000..58cd6b5e6 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/doc/template/Vera.ttf differ diff --git a/import-layers/yocto-poky/bitbake/doc/template/Vera.xml b/import-layers/yocto-poky/bitbake/doc/template/Vera.xml new file mode 100644 index 000000000..3c82043e3 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/Vera.xml @@ -0,0 +1 @@ +BitstreamVeraSans729546928-235-183-23512879283200TYPE0CIDFontType20 \ No newline at end of file diff --git a/import-layers/yocto-poky/bitbake/doc/template/VeraMoBd.ttf b/import-layers/yocto-poky/bitbake/doc/template/VeraMoBd.ttf new file mode 100644 index 000000000..9be6547ed Binary files /dev/null and b/import-layers/yocto-poky/bitbake/doc/template/VeraMoBd.ttf differ diff --git a/import-layers/yocto-poky/bitbake/doc/template/VeraMoBd.xml b/import-layers/yocto-poky/bitbake/doc/template/VeraMoBd.xml new file mode 100644 index 000000000..9b33107a4 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/VeraMoBd.xml @@ -0,0 +1 @@ +BitstreamVeraSansMono-BoldBitstream Vera Sans Mono BoldBitstream Vera Sans Mono729546759-240-19-2356059283400TYPE0CIDFontType20 \ No newline at end of file diff --git a/import-layers/yocto-poky/bitbake/doc/template/VeraMono.ttf b/import-layers/yocto-poky/bitbake/doc/template/VeraMono.ttf new file mode 100644 index 000000000..139f0b431 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/doc/template/VeraMono.ttf differ diff --git a/import-layers/yocto-poky/bitbake/doc/template/VeraMono.xml b/import-layers/yocto-poky/bitbake/doc/template/VeraMono.xml new file mode 100644 index 000000000..3a0a86659 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/VeraMono.xml @@ -0,0 +1 @@ +BitstreamVeraSansMono-RomanBitstream Vera Sans MonoBitstream Vera Sans Mono729546759-240-4-2356059283400TYPE0CIDFontType20 \ No newline at end of file diff --git a/import-layers/yocto-poky/bitbake/doc/template/component.title.xsl b/import-layers/yocto-poky/bitbake/doc/template/component.title.xsl new file mode 100644 index 000000000..faef04326 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/component.title.xsl @@ -0,0 +1,39 @@ + + + + + + + + + + + 6 + 5 + 4 + 3 + 2 + 1 + + + + title + + + + + + + + + + + + + + + diff --git a/import-layers/yocto-poky/bitbake/doc/template/db-pdf.xsl b/import-layers/yocto-poky/bitbake/doc/template/db-pdf.xsl new file mode 100644 index 000000000..3dd065a57 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/db-pdf.xsl @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + 1 10 1 + + + + + + 0.5pt + solid + #cccccc + + + + + + 0.5pt + solid + #cccccc + + + + + #cccccc + + + + #cccccc + + + + + + + + + + + + + diff --git a/import-layers/yocto-poky/bitbake/doc/template/division.title.xsl b/import-layers/yocto-poky/bitbake/doc/template/division.title.xsl new file mode 100644 index 000000000..9c843bc7c --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/division.title.xsl @@ -0,0 +1,25 @@ + + + + + +

+ title + + + + + + + + + + +

+ + + diff --git a/import-layers/yocto-poky/bitbake/doc/template/draft.png b/import-layers/yocto-poky/bitbake/doc/template/draft.png new file mode 100644 index 000000000..53051a9dd Binary files /dev/null and b/import-layers/yocto-poky/bitbake/doc/template/draft.png differ diff --git a/import-layers/yocto-poky/bitbake/doc/template/fop-config.xml b/import-layers/yocto-poky/bitbake/doc/template/fop-config.xml new file mode 100644 index 000000000..09cc5ca0f --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/fop-config.xml @@ -0,0 +1,58 @@ + + + + true + + + true + + + ../template + ../template + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/import-layers/yocto-poky/bitbake/doc/template/formal.object.heading.xsl b/import-layers/yocto-poky/bitbake/doc/template/formal.object.heading.xsl new file mode 100644 index 000000000..4f3900d16 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/formal.object.heading.xsl @@ -0,0 +1,21 @@ + + + + + + + + + +

+ + + + +

+ + \ No newline at end of file diff --git a/import-layers/yocto-poky/bitbake/doc/template/gloss-permalinks.xsl b/import-layers/yocto-poky/bitbake/doc/template/gloss-permalinks.xsl new file mode 100644 index 000000000..6bf58116f --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/gloss-permalinks.xsl @@ -0,0 +1,14 @@ + + + + + + + + + + + diff --git a/import-layers/yocto-poky/bitbake/doc/template/permalinks.xsl b/import-layers/yocto-poky/bitbake/doc/template/permalinks.xsl new file mode 100644 index 000000000..d2a1c1452 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/permalinks.xsl @@ -0,0 +1,25 @@ + + + + + ¶ + + + + + + + + + + + + + + + + + + diff --git a/import-layers/yocto-poky/bitbake/doc/template/section.title.xsl b/import-layers/yocto-poky/bitbake/doc/template/section.title.xsl new file mode 100644 index 000000000..5c6ff9a96 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/section.title.xsl @@ -0,0 +1,55 @@ + + + + + + + + 1 + 2 + 3 + 4 + 5 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/import-layers/yocto-poky/bitbake/doc/template/titlepage.templates.xml b/import-layers/yocto-poky/bitbake/doc/template/titlepage.templates.xml new file mode 100644 index 000000000..38ec11a4c --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/template/titlepage.templates.xml @@ -0,0 +1,1259 @@ + + + + + + + + + + + + +]> + + + + + + + + + + + + + + + <subtitle param:node="ancestor-or-self::article[1]" + keep-with-next="always" + font-size="&hsize3;" + font-weight="bold" + space-after="0.8em"/> + + <corpauthor space-before="0.5em" + font-size="&hsize3;"/> + <authorgroup space-before="0.5em" + font-size="&hsize2;"/> + <author space-before="0.5em" + font-size="&hsize2;" + space-after="0.8em"/> + + <email font-size="&hsize2;"/> + + <othercredit space-before="0.5em"/> + <releaseinfo space-before="0.5em"/> + <copyright space-before="0.5em"/> + <legalnotice text-align="start" + margin-left="0.5in" + margin-right="0.5in" + font-family="{$body.fontset}"/> + <pubdate space-before="0.5em"/> + <para></para> + <revision space-before="0.5em"/> + <revhistory space-before="0.5em"/> + <abstract space-before="0.5em" + text-align="start" + margin-left="0.5in" + margin-right="0.5in" + font-family="{$body.fontset}"/> + + <para></para> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + +<t:titlepage t:element="set" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:named-template="division.title" + param:node="ancestor-or-self::set[1]" + text-align="center" + font-size="&hsize5;" + space-before="&hsize5space;" + font-weight="bold" + font-family="{$title.fontset}"/> + <subtitle + font-family="{$title.fontset}" + text-align="center"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="book" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + + <mediaobject/> + +<!-- + +# If you leave this block of code in then the text title in the +# <title>BitBake User Manual statement of the +# bitbake-user-manual.xml file is rendered on the title page below the +# image. Commenting it out gets it out of there yet allows it +# to be retained in the tab text for the HTML version of the +# manual. + + +--> + <subtitle + text-align="center" + font-size="&hsize4;" + space-before="&hsize4space;" + font-family="{$title.fontset}"/> + <corpauthor font-size="&hsize3;" + keep-with-next="always" + space-before="2in"/> + <authorgroup space-before="2in"/> + <author font-size="&hsize3;" + space-before="&hsize2space;" + keep-with-next="always"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> +<!-- +# If you leave this block of code in then the text title in the +# <title>BitBake User Manual statement of the +# bitbake-user-manual.xml file is rendered on the title page below the +# image. Commenting it out gets it out of there yet allows it +# to be retained in the tab text for the HTML version of the +# manual. + + +--> + <corpauthor/> + <authorgroup t:named-template="verso.authorgroup"/> + <author/> + <othercredit/> + <pubdate space-before="1em"/> + <copyright/> + <abstract/> + <legalnotice font-size="8pt"/> + </t:titlepage-content> + + <t:titlepage-separator> + <fo:block break-after="page"/> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + <fo:block break-after="page"/> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + +<t:titlepage t:element="part" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:named-template="division.title" + param:node="ancestor-or-self::part[1]" + text-align="center" + font-size="&hsize5;" + space-before="&hsize5space;" + font-weight="bold" + font-family="{$title.fontset}"/> + <subtitle + text-align="center" + font-size="&hsize4;" + space-before="&hsize4space;" + font-weight='bold' + font-style='italic' + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<t:titlepage t:element="partintro" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + text-align="center" + font-size="&hsize5;" + font-weight="bold" + space-before="1em" + font-family="{$title.fontset}"/> + <subtitle + text-align="center" + font-size="&hsize2;" + font-weight="bold" + font-style="italic" + font-family="{$title.fontset}"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + +<t:titlepage t:element="reference" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:named-template="division.title" + param:node="ancestor-or-self::reference[1]" + text-align="center" + font-size="&hsize5;" + space-before="&hsize5space;" + font-weight="bold" + font-family="{$title.fontset}"/> + <subtitle + font-family="{$title.fontset}" + text-align="center"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + +<t:titlepage t:element="refsynopsisdiv" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + +<t:titlepage t:element="refsection" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + +<t:titlepage t:element="refsect1" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + +<t:titlepage t:element="refsect2" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + +<t:titlepage t:element="refsect3" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="dedication" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="component.title" + param:node="ancestor-or-self::dedication[1]" + margin-left="{$title.margin.left}" + font-size="&hsize5;" + font-family="{$title.fontset}" + font-weight="bold"/> + <subtitle + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="preface" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="component.title" + param:node="ancestor-or-self::preface[1]" + margin-left="{$title.margin.left}" + font-size="&hsize5;" + font-family="{$title.fontset}" + font-weight="bold"/> + <subtitle + font-family="{$title.fontset}"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="chapter" t:wrapper="fo:block" + font-family="{$title.fontset}"> + <t:titlepage-content t:side="recto" margin-left="{$title.margin.left}"> + <title t:named-template="component.title" + param:node="ancestor-or-self::chapter[1]" + font-size="&hsize5;" + font-weight="bold"/> + + <subtitle space-before="0.5em" + font-style="italic" + font-size="&hsize2;" + font-weight="bold"/> + + <corpauthor space-before="0.5em" + space-after="0.5em" + font-size="&hsize2;"/> + + <authorgroup space-before="0.5em" + space-after="0.5em" + font-size="&hsize2;"/> + + <author space-before="0.5em" + space-after="0.5em" + font-size="&hsize2;"/> + + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="appendix" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:named-template="component.title" + param:node="ancestor-or-self::appendix[1]" + margin-left="{$title.margin.left}" + font-size="&hsize5;" + font-weight="bold" + font-family="{$title.fontset}"/> + <subtitle + font-family="{$title.fontset}"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + +<t:titlepage t:element="section" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + margin-left="{$title.margin.left}" + font-family="{$title.fontset}"/> + <subtitle + font-family="{$title.fontset}"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<t:titlepage t:element="sect1" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + margin-left="{$title.margin.left}" + font-family="{$title.fontset}"/> + <subtitle + font-family="{$title.fontset}"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<t:titlepage t:element="sect2" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + margin-left="{$title.margin.left}" + font-family="{$title.fontset}"/> + <subtitle + font-family="{$title.fontset}"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<t:titlepage t:element="sect3" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + margin-left="{$title.margin.left}" + font-family="{$title.fontset}"/> + <subtitle + font-family="{$title.fontset}"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<t:titlepage t:element="sect4" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + margin-left="{$title.margin.left}" + font-family="{$title.fontset}"/> + <subtitle + font-family="{$title.fontset}"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<t:titlepage t:element="sect5" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + margin-left="{$title.margin.left}" + font-family="{$title.fontset}"/> + <subtitle + font-family="{$title.fontset}"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<t:titlepage t:element="simplesect" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + margin-left="{$title.margin.left}" + font-family="{$title.fontset}"/> + <subtitle + font-family="{$title.fontset}"/> + <corpauthor/> + <authorgroup/> + <author/> + <othercredit/> + <releaseinfo/> + <copyright/> + <legalnotice/> + <pubdate/> + <revision/> + <revhistory/> + <abstract/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="bibliography" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="component.title" + param:node="ancestor-or-self::bibliography[1]" + margin-left="{$title.margin.left}" + font-size="&hsize5;" + font-family="{$title.fontset}" + font-weight="bold"/> + <subtitle + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="bibliodiv" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title t:named-template="component.title" + param:node="ancestor-or-self::bibliodiv[1]" + margin-left="{$title.margin.left}" + font-size="&hsize4;" + font-family="{$title.fontset}" + font-weight="bold"/> + <subtitle + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="glossary" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="component.title" + param:node="ancestor-or-self::glossary[1]" + margin-left="{$title.margin.left}" + font-size="&hsize5;" + font-family="{$title.fontset}" + font-weight="bold"/> + <subtitle + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="glossdiv" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title t:named-template="component.title" + param:node="ancestor-or-self::glossdiv[1]" + margin-left="{$title.margin.left}" + font-size="&hsize4;" + font-family="{$title.fontset}" + font-weight="bold"/> + <subtitle + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="index" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="component.title" + param:node="ancestor-or-self::index[1]" + param:pagewide="1" + margin-left="0pt" + font-size="&hsize5;" + font-family="{$title.fontset}" + font-weight="bold"/> + <subtitle + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + +<!-- ==================================================================== --> + + <!-- The indexdiv.title template is used so that manual and --> + <!-- automatically generated indexdiv titles get the same --> + <!-- formatting. --> + + <t:titlepage t:element="indexdiv" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title t:force="1" + t:named-template="indexdiv.title" + param:title="title"/> + <subtitle + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="setindex" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="component.title" + param:node="ancestor-or-self::setindex[1]" + param:pagewide="1" + margin-left="0pt" + font-size="&hsize5;" + font-family="{$title.fontset}" + font-weight="bold"/> + <subtitle + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="colophon" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="component.title" + param:node="ancestor-or-self::colophon[1]" + margin-left="{$title.margin.left}" + font-size="&hsize5;" + font-family="{$title.fontset}" + font-weight="bold"/> + <subtitle + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> +</t:titlepage> + +<!-- ==================================================================== --> + + <t:titlepage t:element="table.of.contents" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="gentext" + param:key="'TableofContents'" + space-before.minimum="1em" + space-before.optimum="1.5em" + space-before.maximum="2em" + space-after="0.5em" + margin-left="{$title.margin.left}" + font-size="&hsize3;" + font-weight="bold" + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + + <t:titlepage t:element="list.of.tables" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="gentext" + param:key="'ListofTables'" + space-before.minimum="1em" + space-before.optimum="1.5em" + space-before.maximum="2em" + space-after="0.5em" + margin-left="{$title.margin.left}" + font-size="&hsize3;" + font-weight="bold" + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + + <t:titlepage t:element="list.of.figures" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="gentext" + param:key="'ListofFigures'" + space-before.minimum="1em" + space-before.optimum="1.5em" + space-before.maximum="2em" + space-after="0.5em" + margin-left="{$title.margin.left}" + font-size="&hsize3;" + font-weight="bold" + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + + <t:titlepage t:element="list.of.examples" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="gentext" + param:key="'ListofExamples'" + space-before.minimum="1em" + space-before.optimum="1.5em" + space-before.maximum="2em" + space-after="0.5em" + margin-left="{$title.margin.left}" + font-size="&hsize3;" + font-weight="bold" + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + + <t:titlepage t:element="list.of.equations" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="gentext" + param:key="'ListofEquations'" + space-before.minimum="1em" + space-before.optimum="1.5em" + space-before.maximum="2em" + space-after="0.5em" + margin-left="{$title.margin.left}" + font-size="&hsize3;" + font-weight="bold" + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + + <t:titlepage t:element="list.of.procedures" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="gentext" + param:key="'ListofProcedures'" + space-before.minimum="1em" + space-before.optimum="1.5em" + space-before.maximum="2em" + space-after="0.5em" + margin-left="{$title.margin.left}" + font-size="&hsize3;" + font-weight="bold" + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + + <t:titlepage t:element="list.of.unknowns" t:wrapper="fo:block"> + <t:titlepage-content t:side="recto"> + <title + t:force="1" + t:named-template="gentext" + param:key="'ListofUnknown'" + space-before.minimum="1em" + space-before.optimum="1.5em" + space-before.maximum="2em" + space-after="0.5em" + margin-left="{$title.margin.left}" + font-size="&hsize3;" + font-weight="bold" + font-family="{$title.fontset}"/> + </t:titlepage-content> + + <t:titlepage-content t:side="verso"> + </t:titlepage-content> + + <t:titlepage-separator> + </t:titlepage-separator> + + <t:titlepage-before t:side="recto"> + </t:titlepage-before> + + <t:titlepage-before t:side="verso"> + </t:titlepage-before> + </t:titlepage> + +<!-- ==================================================================== --> + +</t:templates> diff --git a/import-layers/yocto-poky/bitbake/doc/tools/docbook-to-pdf b/import-layers/yocto-poky/bitbake/doc/tools/docbook-to-pdf new file mode 100755 index 000000000..558ded9e0 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/doc/tools/docbook-to-pdf @@ -0,0 +1,51 @@ +#!/bin/sh + +if [ -z "$1" -o -z "$2" ]; then + echo "usage: [-v] $0 <docbook file> <templatedir>" + echo + echo "*NOTE* you need xsltproc, fop and nwalsh docbook stylesheets" + echo " installed for this to work!" + echo + exit 0 +fi + +FO=`echo $1 | sed s/.xml/.fo/` || exit 1 +PDF=`echo $1 | sed s/.xml/.pdf/` || exit 1 +TEMPLATEDIR=$2 + +## +# These URI should be rewritten by your distribution's xml catalog to +# match your localy installed XSL stylesheets. +XSL_BASE_URI="http://docbook.sourceforge.net/release/xsl/current" + +# Creates a temporary XSL stylesheet based on titlepage.xsl +xsltproc -o /tmp/titlepage.xsl \ + --xinclude \ + $XSL_BASE_URI/template/titlepage.xsl \ + $TEMPLATEDIR/titlepage.templates.xml || exit 1 + +# Creates the file needed for FOP +xsltproc --xinclude \ + --stringparam hyphenate false \ + --stringparam formal.title.placement "figure after" \ + --stringparam ulink.show 1 \ + --stringparam body.font.master 9 \ + --stringparam title.font.master 11 \ + --stringparam draft.watermark.image "$TEMPLATEDIR/draft.png" \ + --stringparam chapter.autolabel 1 \ + --stringparam appendix.autolabel A \ + --stringparam section.autolabel 1 \ + --stringparam section.label.includes.component.label 1 \ + --output $FO \ + $TEMPLATEDIR/db-pdf.xsl \ + $1 || exit 1 + +# Invokes the Java version of FOP. Uses the additional configuration file common/fop-config.xml +fop -c $TEMPLATEDIR/fop-config.xml -fo $FO -pdf $PDF || exit 1 + +rm -f $FO +rm -f /tmp/titlepage.xsl + +echo +echo " #### Success! $PDF ready. ####" +echo diff --git a/import-layers/yocto-poky/bitbake/lib/bb/COW.py b/import-layers/yocto-poky/bitbake/lib/bb/COW.py new file mode 100644 index 000000000..6917ec378 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/COW.py @@ -0,0 +1,323 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# This is a copy on write dictionary and set which abuses classes to try and be nice and fast. +# +# Copyright (C) 2006 Tim Amsell +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +#Please Note: +# Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW. +# Assign a file to __warn__ to get warnings about slow operations. +# + +from __future__ import print_function +import copy +import types +ImmutableTypes = ( + types.NoneType, + bool, + complex, + float, + int, + long, + tuple, + frozenset, + basestring +) + +MUTABLE = "__mutable__" + +class COWMeta(type): + pass + +class COWDictMeta(COWMeta): + __warn__ = False + __hasmutable__ = False + __marker__ = tuple() + + def __str__(cls): + # FIXME: I have magic numbers! + return "<COWDict Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3) + __repr__ = __str__ + + def cow(cls): + class C(cls): + __count__ = cls.__count__ + 1 + return C + copy = cow + __call__ = cow + + def __setitem__(cls, key, value): + if not isinstance(value, ImmutableTypes): + if not isinstance(value, COWMeta): + cls.__hasmutable__ = True + key += MUTABLE + setattr(cls, key, value) + + def __getmutable__(cls, key, readonly=False): + nkey = key + MUTABLE + try: + return cls.__dict__[nkey] + except KeyError: + pass + + value = getattr(cls, nkey) + if readonly: + return value + + if not cls.__warn__ is False and not isinstance(value, COWMeta): + print("Warning: Doing a copy because %s is a mutable type." % key, file=cls.__warn__) + try: + value = value.copy() + except AttributeError as e: + value = copy.copy(value) + setattr(cls, nkey, value) + return value + + __getmarker__ = [] + def __getreadonly__(cls, key, default=__getmarker__): + """\ + Get a value (even if mutable) which you promise not to change. + """ + return cls.__getitem__(key, default, True) + + def __getitem__(cls, key, default=__getmarker__, readonly=False): + try: + try: + value = getattr(cls, key) + except AttributeError: + value = cls.__getmutable__(key, readonly) + + # This is for values which have been deleted + if value is cls.__marker__: + raise AttributeError("key %s does not exist." % key) + + return value + except AttributeError as e: + if not default is cls.__getmarker__: + return default + + raise KeyError(str(e)) + + def __delitem__(cls, key): + cls.__setitem__(key, cls.__marker__) + + def __revertitem__(cls, key): + if not cls.__dict__.has_key(key): + key += MUTABLE + delattr(cls, key) + + def __contains__(cls, key): + return cls.has_key(key) + + def has_key(cls, key): + value = cls.__getreadonly__(key, cls.__marker__) + if value is cls.__marker__: + return False + return True + + def iter(cls, type, readonly=False): + for key in dir(cls): + if key.startswith("__"): + continue + + if key.endswith(MUTABLE): + key = key[:-len(MUTABLE)] + + if type == "keys": + yield key + + try: + if readonly: + value = cls.__getreadonly__(key) + else: + value = cls[key] + except KeyError: + continue + + if type == "values": + yield value + if type == "items": + yield (key, value) + raise StopIteration() + + def iterkeys(cls): + return cls.iter("keys") + def itervalues(cls, readonly=False): + if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: + print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) + return cls.iter("values", readonly) + def iteritems(cls, readonly=False): + if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: + print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) + return cls.iter("items", readonly) + +class COWSetMeta(COWDictMeta): + def __str__(cls): + # FIXME: I have magic numbers! + return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) -3) + __repr__ = __str__ + + def cow(cls): + class C(cls): + __count__ = cls.__count__ + 1 + return C + + def add(cls, value): + COWDictMeta.__setitem__(cls, repr(hash(value)), value) + + def remove(cls, value): + COWDictMeta.__delitem__(cls, repr(hash(value))) + + def __in__(cls, value): + return COWDictMeta.has_key(repr(hash(value))) + + def iterkeys(cls): + raise TypeError("sets don't have keys") + + def iteritems(cls): + raise TypeError("sets don't have 'items'") + +# These are the actual classes you use! +class COWDictBase(object): + __metaclass__ = COWDictMeta + __count__ = 0 + +class COWSetBase(object): + __metaclass__ = COWSetMeta + __count__ = 0 + +if __name__ == "__main__": + import sys + COWDictBase.__warn__ = sys.stderr + a = COWDictBase() + print("a", a) + + a['a'] = 'a' + a['b'] = 'b' + a['dict'] = {} + + b = a.copy() + print("b", b) + b['c'] = 'b' + + print() + + print("a", a) + for x in a.iteritems(): + print(x) + print("--") + print("b", b) + for x in b.iteritems(): + print(x) + print() + + b['dict']['a'] = 'b' + b['a'] = 'c' + + print("a", a) + for x in a.iteritems(): + print(x) + print("--") + print("b", b) + for x in b.iteritems(): + print(x) + print() + + try: + b['dict2'] + except KeyError as e: + print("Okay!") + + a['set'] = COWSetBase() + a['set'].add("o1") + a['set'].add("o1") + a['set'].add("o2") + + print("a", a) + for x in a['set'].itervalues(): + print(x) + print("--") + print("b", b) + for x in b['set'].itervalues(): + print(x) + print() + + b['set'].add('o3') + + print("a", a) + for x in a['set'].itervalues(): + print(x) + print("--") + print("b", b) + for x in b['set'].itervalues(): + print(x) + print() + + a['set2'] = set() + a['set2'].add("o1") + a['set2'].add("o1") + a['set2'].add("o2") + + print("a", a) + for x in a.iteritems(): + print(x) + print("--") + print("b", b) + for x in b.iteritems(readonly=True): + print(x) + print() + + del b['b'] + try: + print(b['b']) + except KeyError: + print("Yay! deleted key raises error") + + if b.has_key('b'): + print("Boo!") + else: + print("Yay - has_key with delete works!") + + print("a", a) + for x in a.iteritems(): + print(x) + print("--") + print("b", b) + for x in b.iteritems(readonly=True): + print(x) + print() + + b.__revertitem__('b') + + print("a", a) + for x in a.iteritems(): + print(x) + print("--") + print("b", b) + for x in b.iteritems(readonly=True): + print(x) + print() + + b.__revertitem__('dict') + print("a", a) + for x in a.iteritems(): + print(x) + print("--") + print("b", b) + for x in b.iteritems(readonly=True): + print(x) + print() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/__init__.py new file mode 100644 index 000000000..502ad839e --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/__init__.py @@ -0,0 +1,144 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Build System Python Library +# +# Copyright (C) 2003 Holger Schurig +# Copyright (C) 2003, 2004 Chris Larson +# +# Based on Gentoo's portage.py. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +__version__ = "1.30.0" + +import sys +if sys.version_info < (2, 7, 3): + raise RuntimeError("Sorry, python 2.7.3 or later is required for this version of bitbake") + + +class BBHandledException(Exception): + """ + The big dilemma for generic bitbake code is what information to give the user + when an exception occurs. Any exception inheriting this base exception class + has already provided information to the user via some 'fired' message type such as + an explicitly fired event using bb.fire, or a bb.error message. If bitbake + encounters an exception derived from this class, no backtrace or other information + will be given to the user, its assumed the earlier event provided the relevant information. + """ + pass + +import os +import logging + + +class NullHandler(logging.Handler): + def emit(self, record): + pass + +Logger = logging.getLoggerClass() +class BBLogger(Logger): + def __init__(self, name): + if name.split(".")[0] == "BitBake": + self.debug = self.bbdebug + Logger.__init__(self, name) + + def bbdebug(self, level, msg, *args, **kwargs): + return self.log(logging.DEBUG - level + 1, msg, *args, **kwargs) + + def plain(self, msg, *args, **kwargs): + return self.log(logging.INFO + 1, msg, *args, **kwargs) + + def verbose(self, msg, *args, **kwargs): + return self.log(logging.INFO - 1, msg, *args, **kwargs) + +logging.raiseExceptions = False +logging.setLoggerClass(BBLogger) + +logger = logging.getLogger("BitBake") +logger.addHandler(NullHandler()) +logger.setLevel(logging.DEBUG - 2) + +mainlogger = logging.getLogger("BitBake.Main") + +# This has to be imported after the setLoggerClass, as the import of bb.msg +# can result in construction of the various loggers. +import bb.msg + +from bb import fetch2 as fetch +sys.modules['bb.fetch'] = sys.modules['bb.fetch2'] + +# Messaging convenience functions +def plain(*args): + mainlogger.plain(''.join(args)) + +def debug(lvl, *args): + if isinstance(lvl, basestring): + mainlogger.warn("Passed invalid debug level '%s' to bb.debug", lvl) + args = (lvl,) + args + lvl = 1 + mainlogger.debug(lvl, ''.join(args)) + +def note(*args): + mainlogger.info(''.join(args)) + +def warn(*args): + mainlogger.warn(''.join(args)) + +def error(*args, **kwargs): + mainlogger.error(''.join(args), extra=kwargs) + +def fatal(*args, **kwargs): + mainlogger.critical(''.join(args), extra=kwargs) + raise BBHandledException() + +def deprecated(func, name=None, advice=""): + """This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emitted + when the function is used.""" + import warnings + + if advice: + advice = ": %s" % advice + if name is None: + name = func.__name__ + + def newFunc(*args, **kwargs): + warnings.warn("Call to deprecated function %s%s." % (name, + advice), + category=DeprecationWarning, + stacklevel=2) + return func(*args, **kwargs) + newFunc.__name__ = func.__name__ + newFunc.__doc__ = func.__doc__ + newFunc.__dict__.update(func.__dict__) + return newFunc + +# For compatibility +def deprecate_import(current, modulename, fromlist, renames = None): + """Import objects from one module into another, wrapping them with a DeprecationWarning""" + import sys + + module = __import__(modulename, fromlist = fromlist) + for position, objname in enumerate(fromlist): + obj = getattr(module, objname) + newobj = deprecated(obj, "{0}.{1}".format(current, objname), + "Please use {0}.{1} instead".format(modulename, objname)) + if renames: + newname = renames[position] + else: + newname = objname + + setattr(sys.modules[current], newname, newobj) + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/build.py b/import-layers/yocto-poky/bitbake/lib/bb/build.py new file mode 100644 index 000000000..db5072cb4 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/build.py @@ -0,0 +1,784 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake 'Build' implementation +# +# Core code for function execution and task handling in the +# BitBake build tools. +# +# Copyright (C) 2003, 2004 Chris Larson +# +# Based on Gentoo's portage.py. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import os +import sys +import logging +import shlex +import glob +import time +import stat +import bb +import bb.msg +import bb.process +from contextlib import nested +from bb import event, utils + +bblogger = logging.getLogger('BitBake') +logger = logging.getLogger('BitBake.Build') + +NULL = open(os.devnull, 'r+') + +__mtime_cache = {} + +def cached_mtime_noerror(f): + if f not in __mtime_cache: + try: + __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] + except OSError: + return 0 + return __mtime_cache[f] + +def reset_cache(): + global __mtime_cache + __mtime_cache = {} + +# When we execute a Python function, we'd like certain things +# in all namespaces, hence we add them to __builtins__. +# If we do not do this and use the exec globals, they will +# not be available to subfunctions. +__builtins__['bb'] = bb +__builtins__['os'] = os + +class FuncFailed(Exception): + def __init__(self, name = None, logfile = None): + self.logfile = logfile + self.name = name + if name: + self.msg = 'Function failed: %s' % name + else: + self.msg = "Function failed" + + def __str__(self): + if self.logfile and os.path.exists(self.logfile): + msg = ("%s (log file is located at %s)" % + (self.msg, self.logfile)) + else: + msg = self.msg + return msg + +class TaskBase(event.Event): + """Base class for task events""" + + def __init__(self, t, logfile, d): + self._task = t + self._package = d.getVar("PF", True) + self.taskfile = d.getVar("FILE", True) + self.taskname = self._task + self.logfile = logfile + self.time = time.time() + event.Event.__init__(self) + self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName()) + + def getTask(self): + return self._task + + def setTask(self, task): + self._task = task + + def getDisplayName(self): + return bb.event.getName(self)[4:] + + task = property(getTask, setTask, None, "task property") + +class TaskStarted(TaskBase): + """Task execution started""" + def __init__(self, t, logfile, taskflags, d): + super(TaskStarted, self).__init__(t, logfile, d) + self.taskflags = taskflags + +class TaskSucceeded(TaskBase): + """Task execution completed""" + +class TaskFailed(TaskBase): + """Task execution failed""" + + def __init__(self, task, logfile, metadata, errprinted = False): + self.errprinted = errprinted + super(TaskFailed, self).__init__(task, logfile, metadata) + +class TaskFailedSilent(TaskBase): + """Task execution failed (silently)""" + def getDisplayName(self): + # Don't need to tell the user it was silent + return "Failed" + +class TaskInvalid(TaskBase): + + def __init__(self, task, metadata): + super(TaskInvalid, self).__init__(task, None, metadata) + self._message = "No such task '%s'" % task + + +class LogTee(object): + def __init__(self, logger, outfile): + self.outfile = outfile + self.logger = logger + self.name = self.outfile.name + + def write(self, string): + self.logger.plain(string) + self.outfile.write(string) + + def __enter__(self): + self.outfile.__enter__() + return self + + def __exit__(self, *excinfo): + self.outfile.__exit__(*excinfo) + + def __repr__(self): + return '<LogTee {0}>'.format(self.name) + def flush(self): + self.outfile.flush() + +# +# pythonexception allows the python exceptions generated to be raised +# as the real exceptions (not FuncFailed) and without a backtrace at the +# origin of the failure. +# +def exec_func(func, d, dirs = None, pythonexception=False): + """Execute a BB 'function'""" + + body = d.getVar(func, False) + if not body: + if body is None: + logger.warn("Function %s doesn't exist", func) + return + + flags = d.getVarFlags(func) + cleandirs = flags.get('cleandirs') + if cleandirs: + for cdir in d.expand(cleandirs).split(): + bb.utils.remove(cdir, True) + bb.utils.mkdirhier(cdir) + + if dirs is None: + dirs = flags.get('dirs') + if dirs: + dirs = d.expand(dirs).split() + + if dirs: + for adir in dirs: + bb.utils.mkdirhier(adir) + adir = dirs[-1] + else: + adir = d.getVar('B', True) + bb.utils.mkdirhier(adir) + + ispython = flags.get('python') + + lockflag = flags.get('lockfiles') + if lockflag: + lockfiles = [f for f in d.expand(lockflag).split()] + else: + lockfiles = None + + tempdir = d.getVar('T', True) + + # or func allows items to be executed outside of the normal + # task set, such as buildhistory + task = d.getVar('BB_RUNTASK', True) or func + if task == func: + taskfunc = task + else: + taskfunc = "%s.%s" % (task, func) + + runfmt = d.getVar('BB_RUNFMT', True) or "run.{func}.{pid}" + runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid()) + runfile = os.path.join(tempdir, runfn) + bb.utils.mkdirhier(os.path.dirname(runfile)) + + # Setup the courtesy link to the runfn, only for tasks + # we create the link 'just' before the run script is created + # if we create it after, and if the run script fails, then the + # link won't be created as an exception would be fired. + if task == func: + runlink = os.path.join(tempdir, 'run.{0}'.format(task)) + if runlink: + bb.utils.remove(runlink) + + try: + os.symlink(runfn, runlink) + except OSError: + pass + + with bb.utils.fileslocked(lockfiles): + if ispython: + exec_func_python(func, d, runfile, cwd=adir, pythonexception=pythonexception) + else: + exec_func_shell(func, d, runfile, cwd=adir) + +_functionfmt = """ +{function}(d) +""" +logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") +def exec_func_python(func, d, runfile, cwd=None, pythonexception=False): + """Execute a python BB 'function'""" + + code = _functionfmt.format(function=func) + bb.utils.mkdirhier(os.path.dirname(runfile)) + with open(runfile, 'w') as script: + bb.data.emit_func_python(func, script, d) + + if cwd: + try: + olddir = os.getcwd() + except OSError: + olddir = None + os.chdir(cwd) + + bb.debug(2, "Executing python function %s" % func) + + try: + text = "def %s(d):\n%s" % (func, d.getVar(func, False)) + fn = d.getVarFlag(func, "filename", False) + lineno = int(d.getVarFlag(func, "lineno", False)) + bb.methodpool.insert_method(func, text, fn, lineno - 1) + + comp = utils.better_compile(code, func, "exec_python_func() autogenerated") + utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated", pythonexception=pythonexception) + except (bb.parse.SkipRecipe, bb.build.FuncFailed): + raise + except: + if pythonexception: + raise + raise FuncFailed(func, None) + finally: + bb.debug(2, "Python function %s finished" % func) + + if cwd and olddir: + try: + os.chdir(olddir) + except OSError: + pass + +def shell_trap_code(): + return '''#!/bin/sh\n +# Emit a useful diagnostic if something fails: +bb_exit_handler() { + ret=$? + case $ret in + 0) ;; + *) case $BASH_VERSION in + "") echo "WARNING: exit code $ret from a shell command.";; + *) echo "WARNING: ${BASH_SOURCE[0]}:${BASH_LINENO[0]} exit $ret from '$BASH_COMMAND'";; + esac + exit $ret + esac +} +trap 'bb_exit_handler' 0 +set -e +''' + +def exec_func_shell(func, d, runfile, cwd=None): + """Execute a shell function from the metadata + + Note on directory behavior. The 'dirs' varflag should contain a list + of the directories you need created prior to execution. The last + item in the list is where we will chdir/cd to. + """ + + # Don't let the emitted shell script override PWD + d.delVarFlag('PWD', 'export') + + with open(runfile, 'w') as script: + script.write(shell_trap_code()) + + bb.data.emit_func(func, script, d) + + if bb.msg.loggerVerboseLogs: + script.write("set -x\n") + if cwd: + script.write("cd '%s'\n" % cwd) + script.write("%s\n" % func) + script.write(''' +# cleanup +ret=$? +trap '' 0 +exit $ret +''') + + os.chmod(runfile, 0775) + + cmd = runfile + if d.getVarFlag(func, 'fakeroot', False): + fakerootcmd = d.getVar('FAKEROOT', True) + if fakerootcmd: + cmd = [fakerootcmd, runfile] + + if bb.msg.loggerDefaultVerbose: + logfile = LogTee(logger, sys.stdout) + else: + logfile = sys.stdout + + def readfifo(data): + lines = data.split('\0') + for line in lines: + splitval = line.split(' ', 1) + cmd = splitval[0] + if len(splitval) > 1: + value = splitval[1] + else: + value = '' + if cmd == 'bbplain': + bb.plain(value) + elif cmd == 'bbnote': + bb.note(value) + elif cmd == 'bbwarn': + bb.warn(value) + elif cmd == 'bberror': + bb.error(value) + elif cmd == 'bbfatal': + # The caller will call exit themselves, so bb.error() is + # what we want here rather than bb.fatal() + bb.error(value) + elif cmd == 'bbfatal_log': + bb.error(value, forcelog=True) + elif cmd == 'bbdebug': + splitval = value.split(' ', 1) + level = int(splitval[0]) + value = splitval[1] + bb.debug(level, value) + + tempdir = d.getVar('T', True) + fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid()) + if os.path.exists(fifopath): + os.unlink(fifopath) + os.mkfifo(fifopath) + with open(fifopath, 'r+') as fifo: + try: + bb.debug(2, "Executing shell function %s" % func) + + try: + with open(os.devnull, 'r+') as stdin: + bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)]) + except bb.process.CmdError: + logfn = d.getVar('BB_LOGFILE', True) + raise FuncFailed(func, logfn) + finally: + os.unlink(fifopath) + + bb.debug(2, "Shell function %s finished" % func) + +def _task_data(fn, task, d): + localdata = bb.data.createCopy(d) + localdata.setVar('BB_FILENAME', fn) + localdata.setVar('BB_CURRENTTASK', task[3:]) + localdata.setVar('OVERRIDES', 'task-%s:%s' % + (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) + localdata.finalize() + bb.data.expandKeys(localdata) + return localdata + +def _exec_task(fn, task, d, quieterr): + """Execute a BB 'task' + + Execution of a task involves a bit more setup than executing a function, + running it with its own local metadata, and with some useful variables set. + """ + if not d.getVarFlag(task, 'task', False): + event.fire(TaskInvalid(task, d), d) + logger.error("No such task: %s" % task) + return 1 + + logger.debug(1, "Executing task %s", task) + + localdata = _task_data(fn, task, d) + tempdir = localdata.getVar('T', True) + if not tempdir: + bb.fatal("T variable not set, unable to build") + + # Change nice level if we're asked to + nice = localdata.getVar("BB_TASK_NICE_LEVEL", True) + if nice: + curnice = os.nice(0) + nice = int(nice) - curnice + newnice = os.nice(nice) + logger.debug(1, "Renice to %s " % newnice) + ionice = localdata.getVar("BB_TASK_IONICE_LEVEL", True) + if ionice: + try: + cls, prio = ionice.split(".", 1) + bb.utils.ioprio_set(os.getpid(), int(cls), int(prio)) + except: + bb.warn("Invalid ionice level %s" % ionice) + + bb.utils.mkdirhier(tempdir) + + # Determine the logfile to generate + logfmt = localdata.getVar('BB_LOGFMT', True) or 'log.{task}.{pid}' + logbase = logfmt.format(task=task, pid=os.getpid()) + + # Document the order of the tasks... + logorder = os.path.join(tempdir, 'log.task_order') + try: + with open(logorder, 'a') as logorderfile: + logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) + except OSError: + logger.exception("Opening log file '%s'", logorder) + pass + + # Setup the courtesy link to the logfn + loglink = os.path.join(tempdir, 'log.{0}'.format(task)) + logfn = os.path.join(tempdir, logbase) + if loglink: + bb.utils.remove(loglink) + + try: + os.symlink(logbase, loglink) + except OSError: + pass + + prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) + postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) + + class ErrorCheckHandler(logging.Handler): + def __init__(self): + self.triggered = False + logging.Handler.__init__(self, logging.ERROR) + def emit(self, record): + if getattr(record, 'forcelog', False): + self.triggered = False + else: + self.triggered = True + + # Handle logfiles + si = open('/dev/null', 'r') + try: + bb.utils.mkdirhier(os.path.dirname(logfn)) + logfile = open(logfn, 'w') + except OSError: + logger.exception("Opening log file '%s'", logfn) + pass + + # Dup the existing fds so we dont lose them + osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] + oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] + ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] + + # Replace those fds with our own + os.dup2(si.fileno(), osi[1]) + os.dup2(logfile.fileno(), oso[1]) + os.dup2(logfile.fileno(), ose[1]) + + # Ensure Python logging goes to the logfile + handler = logging.StreamHandler(logfile) + handler.setFormatter(logformatter) + # Always enable full debug output into task logfiles + handler.setLevel(logging.DEBUG - 2) + bblogger.addHandler(handler) + + errchk = ErrorCheckHandler() + bblogger.addHandler(errchk) + + localdata.setVar('BB_LOGFILE', logfn) + localdata.setVar('BB_RUNTASK', task) + + flags = localdata.getVarFlags(task) + + event.fire(TaskStarted(task, logfn, flags, localdata), localdata) + try: + for func in (prefuncs or '').split(): + exec_func(func, localdata) + exec_func(task, localdata) + for func in (postfuncs or '').split(): + exec_func(func, localdata) + except FuncFailed as exc: + if quieterr: + event.fire(TaskFailedSilent(task, logfn, localdata), localdata) + else: + errprinted = errchk.triggered + logger.error(str(exc)) + event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata) + return 1 + finally: + sys.stdout.flush() + sys.stderr.flush() + + bblogger.removeHandler(handler) + + # Restore the backup fds + os.dup2(osi[0], osi[1]) + os.dup2(oso[0], oso[1]) + os.dup2(ose[0], ose[1]) + + # Close the backup fds + os.close(osi[0]) + os.close(oso[0]) + os.close(ose[0]) + si.close() + + logfile.close() + if os.path.exists(logfn) and os.path.getsize(logfn) == 0: + logger.debug(2, "Zero size logfn %s, removing", logfn) + bb.utils.remove(logfn) + bb.utils.remove(loglink) + event.fire(TaskSucceeded(task, logfn, localdata), localdata) + + if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): + make_stamp(task, localdata) + + return 0 + +def exec_task(fn, task, d, profile = False): + try: + quieterr = False + if d.getVarFlag(task, "quieterrors", False) is not None: + quieterr = True + + if profile: + profname = "profile-%s.log" % (d.getVar("PN", True) + "-" + task) + try: + import cProfile as profile + except: + import profile + prof = profile.Profile() + ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr) + prof.dump_stats(profname) + bb.utils.process_profilelog(profname) + + return ret + else: + return _exec_task(fn, task, d, quieterr) + + except Exception: + from traceback import format_exc + if not quieterr: + logger.error("Build of %s failed" % (task)) + logger.error(format_exc()) + failedevent = TaskFailed(task, None, d, True) + event.fire(failedevent, d) + return 1 + +def stamp_internal(taskname, d, file_name, baseonly=False): + """ + Internal stamp helper function + Makes sure the stamp directory exists + Returns the stamp path+filename + + In the bitbake core, d can be a CacheData and file_name will be set. + When called in task context, d will be a data store, file_name will not be set + """ + taskflagname = taskname + if taskname.endswith("_setscene") and taskname != "do_setscene": + taskflagname = taskname.replace("_setscene", "") + + if file_name: + stamp = d.stamp[file_name] + extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" + else: + stamp = d.getVar('STAMP', True) + file_name = d.getVar('BB_FILENAME', True) + extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or "" + + if baseonly: + return stamp + + if not stamp: + return + + stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo) + + stampdir = os.path.dirname(stamp) + if cached_mtime_noerror(stampdir) == 0: + bb.utils.mkdirhier(stampdir) + + return stamp + +def stamp_cleanmask_internal(taskname, d, file_name): + """ + Internal stamp helper function to generate stamp cleaning mask + Returns the stamp path+filename + + In the bitbake core, d can be a CacheData and file_name will be set. + When called in task context, d will be a data store, file_name will not be set + """ + taskflagname = taskname + if taskname.endswith("_setscene") and taskname != "do_setscene": + taskflagname = taskname.replace("_setscene", "") + + if file_name: + stamp = d.stampclean[file_name] + extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" + else: + stamp = d.getVar('STAMPCLEAN', True) + file_name = d.getVar('BB_FILENAME', True) + extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or "" + + if not stamp: + return [] + + cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo) + + return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] + +def make_stamp(task, d, file_name = None): + """ + Creates/updates a stamp for a given task + (d can be a data dict or dataCache) + """ + cleanmask = stamp_cleanmask_internal(task, d, file_name) + for mask in cleanmask: + for name in glob.glob(mask): + # Preserve sigdata files in the stamps directory + if "sigdata" in name: + continue + # Preserve taint files in the stamps directory + if name.endswith('.taint'): + continue + os.unlink(name) + + stamp = stamp_internal(task, d, file_name) + # Remove the file and recreate to force timestamp + # change on broken NFS filesystems + if stamp: + bb.utils.remove(stamp) + open(stamp, "w").close() + + # If we're in task context, write out a signature file for each task + # as it completes + if not task.endswith("_setscene") and task != "do_setscene" and not file_name: + stampbase = stamp_internal(task, d, None, True) + file_name = d.getVar('BB_FILENAME', True) + bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True) + +def del_stamp(task, d, file_name = None): + """ + Removes a stamp for a given task + (d can be a data dict or dataCache) + """ + stamp = stamp_internal(task, d, file_name) + bb.utils.remove(stamp) + +def write_taint(task, d, file_name = None): + """ + Creates a "taint" file which will force the specified task and its + dependents to be re-run the next time by influencing the value of its + taskhash. + (d can be a data dict or dataCache) + """ + import uuid + if file_name: + taintfn = d.stamp[file_name] + '.' + task + '.taint' + else: + taintfn = d.getVar('STAMP', True) + '.' + task + '.taint' + bb.utils.mkdirhier(os.path.dirname(taintfn)) + # The specific content of the taint file is not really important, + # we just need it to be random, so a random UUID is used + with open(taintfn, 'w') as taintf: + taintf.write(str(uuid.uuid4())) + +def stampfile(taskname, d, file_name = None): + """ + Return the stamp for a given task + (d can be a data dict or dataCache) + """ + return stamp_internal(taskname, d, file_name) + +def add_tasks(tasklist, d): + task_deps = d.getVar('_task_deps', False) + if not task_deps: + task_deps = {} + if not 'tasks' in task_deps: + task_deps['tasks'] = [] + if not 'parents' in task_deps: + task_deps['parents'] = {} + + for task in tasklist: + task = d.expand(task) + + d.setVarFlag(task, 'task', 1) + + if not task in task_deps['tasks']: + task_deps['tasks'].append(task) + + flags = d.getVarFlags(task) + def getTask(name): + if not name in task_deps: + task_deps[name] = {} + if name in flags: + deptask = d.expand(flags[name]) + task_deps[name][task] = deptask + getTask('depends') + getTask('rdepends') + getTask('deptask') + getTask('rdeptask') + getTask('recrdeptask') + getTask('recideptask') + getTask('nostamp') + getTask('fakeroot') + getTask('noexec') + getTask('umask') + task_deps['parents'][task] = [] + if 'deps' in flags: + for dep in flags['deps']: + dep = d.expand(dep) + task_deps['parents'][task].append(dep) + + # don't assume holding a reference + d.setVar('_task_deps', task_deps) + +def addtask(task, before, after, d): + if task[:3] != "do_": + task = "do_" + task + + d.setVarFlag(task, "task", 1) + bbtasks = d.getVar('__BBTASKS', False) or [] + if task not in bbtasks: + bbtasks.append(task) + d.setVar('__BBTASKS', bbtasks) + + existing = d.getVarFlag(task, "deps", False) or [] + if after is not None: + # set up deps for function + for entry in after.split(): + if entry not in existing: + existing.append(entry) + d.setVarFlag(task, "deps", existing) + if before is not None: + # set up things that depend on this func + for entry in before.split(): + existing = d.getVarFlag(entry, "deps", False) or [] + if task not in existing: + d.setVarFlag(entry, "deps", [task] + existing) + +def deltask(task, d): + if task[:3] != "do_": + task = "do_" + task + + bbtasks = d.getVar('__BBTASKS', False) or [] + if task in bbtasks: + bbtasks.remove(task) + d.setVar('__BBTASKS', bbtasks) + + d.delVarFlag(task, 'deps') + for bbtask in d.getVar('__BBTASKS', False) or []: + deps = d.getVarFlag(bbtask, 'deps', False) or [] + if task in deps: + deps.remove(task) + d.setVarFlag(bbtask, 'deps', deps) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cache.py b/import-layers/yocto-poky/bitbake/lib/bb/cache.py new file mode 100644 index 000000000..af5b9fbc6 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/cache.py @@ -0,0 +1,849 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Cache implementation +# +# Caching of bitbake variables before task execution + +# Copyright (C) 2006 Richard Purdie +# Copyright (C) 2012 Intel Corporation + +# but small sections based on code from bin/bitbake: +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import logging +from collections import defaultdict +import bb.utils + +logger = logging.getLogger("BitBake.Cache") + +try: + import cPickle as pickle +except ImportError: + import pickle + logger.info("Importing cPickle failed. " + "Falling back to a very slow implementation.") + +__cache_version__ = "149" + +def getCacheFile(path, filename, data_hash): + return os.path.join(path, filename + "." + data_hash) + +# RecipeInfoCommon defines common data retrieving methods +# from meta data for caches. CoreRecipeInfo as well as other +# Extra RecipeInfo needs to inherit this class +class RecipeInfoCommon(object): + + @classmethod + def listvar(cls, var, metadata): + return cls.getvar(var, metadata).split() + + @classmethod + def intvar(cls, var, metadata): + return int(cls.getvar(var, metadata) or 0) + + @classmethod + def depvar(cls, var, metadata): + return bb.utils.explode_deps(cls.getvar(var, metadata)) + + @classmethod + def pkgvar(cls, var, packages, metadata): + return dict((pkg, cls.depvar("%s_%s" % (var, pkg), metadata)) + for pkg in packages) + + @classmethod + def taskvar(cls, var, tasks, metadata): + return dict((task, cls.getvar("%s_task-%s" % (var, task), metadata)) + for task in tasks) + + @classmethod + def flaglist(cls, flag, varlist, metadata, squash=False): + out_dict = dict((var, metadata.getVarFlag(var, flag, True)) + for var in varlist) + if squash: + return dict((k,v) for (k,v) in out_dict.iteritems() if v) + else: + return out_dict + + @classmethod + def getvar(cls, var, metadata, expand = True): + return metadata.getVar(var, expand) or '' + + +class CoreRecipeInfo(RecipeInfoCommon): + __slots__ = () + + cachefile = "bb_cache.dat" + + def __init__(self, filename, metadata): + self.file_depends = metadata.getVar('__depends', False) + self.timestamp = bb.parse.cached_mtime(filename) + self.variants = self.listvar('__VARIANTS', metadata) + [''] + self.appends = self.listvar('__BBAPPEND', metadata) + self.nocache = self.getvar('BB_DONT_CACHE', metadata) + + self.skipreason = self.getvar('__SKIPPED', metadata) + if self.skipreason: + self.pn = self.getvar('PN', metadata) or bb.parse.BBHandler.vars_from_file(filename,metadata)[0] + self.skipped = True + self.provides = self.depvar('PROVIDES', metadata) + self.rprovides = self.depvar('RPROVIDES', metadata) + return + + self.tasks = metadata.getVar('__BBTASKS', False) + + self.pn = self.getvar('PN', metadata) + self.packages = self.listvar('PACKAGES', metadata) + if not self.pn in self.packages: + self.packages.append(self.pn) + + self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata) + self.hashfilename = self.getvar('BB_HASHFILENAME', metadata) + + self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}} + + self.skipped = False + self.pe = self.getvar('PE', metadata) + self.pv = self.getvar('PV', metadata) + self.pr = self.getvar('PR', metadata) + self.defaultpref = self.intvar('DEFAULT_PREFERENCE', metadata) + self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata) + self.stamp = self.getvar('STAMP', metadata) + self.stampclean = self.getvar('STAMPCLEAN', metadata) + self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata) + self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True) + self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata) + self.depends = self.depvar('DEPENDS', metadata) + self.provides = self.depvar('PROVIDES', metadata) + self.rdepends = self.depvar('RDEPENDS', metadata) + self.rprovides = self.depvar('RPROVIDES', metadata) + self.rrecommends = self.depvar('RRECOMMENDS', metadata) + self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata) + self.rdepends_pkg = self.pkgvar('RDEPENDS', self.packages, metadata) + self.rrecommends_pkg = self.pkgvar('RRECOMMENDS', self.packages, metadata) + self.inherits = self.getvar('__inherit_cache', metadata, expand=False) + self.fakerootenv = self.getvar('FAKEROOTENV', metadata) + self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata) + self.fakerootnoenv = self.getvar('FAKEROOTNOENV', metadata) + self.extradepsfunc = self.getvar('calculate_extra_depends', metadata) + + @classmethod + def init_cacheData(cls, cachedata): + # CacheData in Core RecipeInfo Class + cachedata.task_deps = {} + cachedata.pkg_fn = {} + cachedata.pkg_pn = defaultdict(list) + cachedata.pkg_pepvpr = {} + cachedata.pkg_dp = {} + + cachedata.stamp = {} + cachedata.stampclean = {} + cachedata.stamp_extrainfo = {} + cachedata.file_checksums = {} + cachedata.fn_provides = {} + cachedata.pn_provides = defaultdict(list) + cachedata.all_depends = [] + + cachedata.deps = defaultdict(list) + cachedata.packages = defaultdict(list) + cachedata.providers = defaultdict(list) + cachedata.rproviders = defaultdict(list) + cachedata.packages_dynamic = defaultdict(list) + + cachedata.rundeps = defaultdict(lambda: defaultdict(list)) + cachedata.runrecs = defaultdict(lambda: defaultdict(list)) + cachedata.possible_world = [] + cachedata.universe_target = [] + cachedata.hashfn = {} + + cachedata.basetaskhash = {} + cachedata.inherits = {} + cachedata.fakerootenv = {} + cachedata.fakerootnoenv = {} + cachedata.fakerootdirs = {} + cachedata.extradepsfunc = {} + + def add_cacheData(self, cachedata, fn): + cachedata.task_deps[fn] = self.task_deps + cachedata.pkg_fn[fn] = self.pn + cachedata.pkg_pn[self.pn].append(fn) + cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr) + cachedata.pkg_dp[fn] = self.defaultpref + cachedata.stamp[fn] = self.stamp + cachedata.stampclean[fn] = self.stampclean + cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo + cachedata.file_checksums[fn] = self.file_checksums + + provides = [self.pn] + for provide in self.provides: + if provide not in provides: + provides.append(provide) + cachedata.fn_provides[fn] = provides + + for provide in provides: + cachedata.providers[provide].append(fn) + if provide not in cachedata.pn_provides[self.pn]: + cachedata.pn_provides[self.pn].append(provide) + + for dep in self.depends: + if dep not in cachedata.deps[fn]: + cachedata.deps[fn].append(dep) + if dep not in cachedata.all_depends: + cachedata.all_depends.append(dep) + + rprovides = self.rprovides + for package in self.packages: + cachedata.packages[package].append(fn) + rprovides += self.rprovides_pkg[package] + + for rprovide in rprovides: + if fn not in cachedata.rproviders[rprovide]: + cachedata.rproviders[rprovide].append(fn) + + for package in self.packages_dynamic: + cachedata.packages_dynamic[package].append(fn) + + # Build hash of runtime depends and recommends + for package in self.packages + [self.pn]: + cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package] + cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package] + + # Collect files we may need for possible world-dep + # calculations + if self.not_world: + logger.debug(1, "EXCLUDE FROM WORLD: %s", fn) + else: + cachedata.possible_world.append(fn) + + # create a collection of all targets for sanity checking + # tasks, such as upstream versions, license, and tools for + # task and image creation. + cachedata.universe_target.append(self.pn) + + cachedata.hashfn[fn] = self.hashfilename + for task, taskhash in self.basetaskhashes.iteritems(): + identifier = '%s.%s' % (fn, task) + cachedata.basetaskhash[identifier] = taskhash + + cachedata.inherits[fn] = self.inherits + cachedata.fakerootenv[fn] = self.fakerootenv + cachedata.fakerootnoenv[fn] = self.fakerootnoenv + cachedata.fakerootdirs[fn] = self.fakerootdirs + cachedata.extradepsfunc[fn] = self.extradepsfunc + + + +class Cache(object): + """ + BitBake Cache implementation + """ + + def __init__(self, data, data_hash, caches_array): + # Pass caches_array information into Cache Constructor + # It will be used later for deciding whether we + # need extra cache file dump/load support + self.caches_array = caches_array + self.cachedir = data.getVar("CACHE", True) + self.clean = set() + self.checked = set() + self.depends_cache = {} + self.data = None + self.data_fn = None + self.cacheclean = True + self.data_hash = data_hash + + if self.cachedir in [None, '']: + self.has_cache = False + logger.info("Not using a cache. " + "Set CACHE = <directory> to enable.") + return + + self.has_cache = True + self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) + + logger.debug(1, "Using cache in '%s'", self.cachedir) + bb.utils.mkdirhier(self.cachedir) + + cache_ok = True + if self.caches_array: + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cache_ok = cache_ok and os.path.exists(cachefile) + cache_class.init_cacheData(self) + if cache_ok: + self.load_cachefile() + elif os.path.isfile(self.cachefile): + logger.info("Out of date cache found, rebuilding...") + + def load_cachefile(self): + # Firstly, using core cache file information for + # valid checking + with open(self.cachefile, "rb") as cachefile: + pickled = pickle.Unpickler(cachefile) + try: + cache_ver = pickled.load() + bitbake_ver = pickled.load() + except Exception: + logger.info('Invalid cache, rebuilding...') + return + + if cache_ver != __cache_version__: + logger.info('Cache version mismatch, rebuilding...') + return + elif bitbake_ver != bb.__version__: + logger.info('Bitbake version mismatch, rebuilding...') + return + + + cachesize = 0 + previous_progress = 0 + previous_percent = 0 + + # Calculate the correct cachesize of all those cache files + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + with open(cachefile, "rb") as cachefile: + cachesize += os.fstat(cachefile.fileno()).st_size + + bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) + + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + with open(cachefile, "rb") as cachefile: + pickled = pickle.Unpickler(cachefile) + while cachefile: + try: + key = pickled.load() + value = pickled.load() + except Exception: + break + if self.depends_cache.has_key(key): + self.depends_cache[key].append(value) + else: + self.depends_cache[key] = [value] + # only fire events on even percentage boundaries + current_progress = cachefile.tell() + previous_progress + current_percent = 100 * current_progress / cachesize + if current_percent > previous_percent: + previous_percent = current_percent + bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), + self.data) + + previous_progress += current_progress + + # Note: depends cache number is corresponding to the parsing file numbers. + # The same file has several caches, still regarded as one item in the cache + bb.event.fire(bb.event.CacheLoadCompleted(cachesize, + len(self.depends_cache)), + self.data) + + + @staticmethod + def virtualfn2realfn(virtualfn): + """ + Convert a virtual file name to a real one + the associated subclass keyword + """ + + fn = virtualfn + cls = "" + if virtualfn.startswith('virtual:'): + elems = virtualfn.split(':') + cls = ":".join(elems[1:-1]) + fn = elems[-1] + return (fn, cls) + + @staticmethod + def realfn2virtual(realfn, cls): + """ + Convert a real filename + the associated subclass keyword to a virtual filename + """ + if cls == "": + return realfn + return "virtual:" + cls + ":" + realfn + + @classmethod + def loadDataFull(cls, virtualfn, appends, cfgData): + """ + Return a complete set of data for fn. + To do this, we need to parse the file. + """ + + (fn, virtual) = cls.virtualfn2realfn(virtualfn) + + logger.debug(1, "Parsing %s (full)", fn) + + cfgData.setVar("__ONLYFINALISE", virtual or "default") + bb_data = cls.load_bbfile(fn, appends, cfgData) + return bb_data[virtual] + + @classmethod + def parse(cls, filename, appends, configdata, caches_array): + """Parse the specified filename, returning the recipe information""" + infos = [] + datastores = cls.load_bbfile(filename, appends, configdata) + depends = [] + for variant, data in sorted(datastores.iteritems(), + key=lambda i: i[0], + reverse=True): + virtualfn = cls.realfn2virtual(filename, variant) + depends = depends + (data.getVar("__depends", False) or []) + if depends and not variant: + data.setVar("__depends", depends) + + info_array = [] + for cache_class in caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + info = cache_class(filename, data) + info_array.append(info) + infos.append((virtualfn, info_array)) + + return infos + + def load(self, filename, appends, configdata): + """Obtain the recipe information for the specified filename, + using cached values if available, otherwise parsing. + + Note that if it does parse to obtain the info, it will not + automatically add the information to the cache or to your + CacheData. Use the add or add_info method to do so after + running this, or use loadData instead.""" + cached = self.cacheValid(filename, appends) + if cached: + infos = [] + # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo] + info_array = self.depends_cache[filename] + for variant in info_array[0].variants: + virtualfn = self.realfn2virtual(filename, variant) + infos.append((virtualfn, self.depends_cache[virtualfn])) + else: + logger.debug(1, "Parsing %s", filename) + return self.parse(filename, appends, configdata, self.caches_array) + + return cached, infos + + def loadData(self, fn, appends, cfgData, cacheData): + """Load the recipe info for the specified filename, + parsing and adding to the cache if necessary, and adding + the recipe information to the supplied CacheData instance.""" + skipped, virtuals = 0, 0 + + cached, infos = self.load(fn, appends, cfgData) + for virtualfn, info_array in infos: + if info_array[0].skipped: + logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) + skipped += 1 + else: + self.add_info(virtualfn, info_array, cacheData, not cached) + virtuals += 1 + + return cached, skipped, virtuals + + def cacheValid(self, fn, appends): + """ + Is the cache valid for fn? + Fast version, no timestamps checked. + """ + if fn not in self.checked: + self.cacheValidUpdate(fn, appends) + + # Is cache enabled? + if not self.has_cache: + return False + if fn in self.clean: + return True + return False + + def cacheValidUpdate(self, fn, appends): + """ + Is the cache valid for fn? + Make thorough (slower) checks including timestamps. + """ + # Is cache enabled? + if not self.has_cache: + return False + + self.checked.add(fn) + + # File isn't in depends_cache + if not fn in self.depends_cache: + logger.debug(2, "Cache: %s is not cached", fn) + return False + + mtime = bb.parse.cached_mtime_noerror(fn) + + # Check file still exists + if mtime == 0: + logger.debug(2, "Cache: %s no longer exists", fn) + self.remove(fn) + return False + + info_array = self.depends_cache[fn] + # Check the file's timestamp + if mtime != info_array[0].timestamp: + logger.debug(2, "Cache: %s changed", fn) + self.remove(fn) + return False + + # Check dependencies are still valid + depends = info_array[0].file_depends + if depends: + for f, old_mtime in depends: + fmtime = bb.parse.cached_mtime_noerror(f) + # Check if file still exists + if old_mtime != 0 and fmtime == 0: + logger.debug(2, "Cache: %s's dependency %s was removed", + fn, f) + self.remove(fn) + return False + + if (fmtime != old_mtime): + logger.debug(2, "Cache: %s's dependency %s changed", + fn, f) + self.remove(fn) + return False + + if hasattr(info_array[0], 'file_checksums'): + for _, fl in info_array[0].file_checksums.items(): + fl = fl.strip() + while fl: + # A .split() would be simpler but means spaces or colons in filenames would break + a = fl.find(":True") + b = fl.find(":False") + if ((a < 0) and b) or ((b > 0) and (b < a)): + f = fl[:b+6] + fl = fl[b+7:] + elif ((b < 0) and a) or ((a > 0) and (a < b)): + f = fl[:a+5] + fl = fl[a+6:] + else: + break + fl = fl.strip() + if "*" in f: + continue + f, exist = f.split(":") + if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): + logger.debug(2, "Cache: %s's file checksum list file %s changed", + fn, f) + self.remove(fn) + return False + + if appends != info_array[0].appends: + logger.debug(2, "Cache: appends for %s changed", fn) + logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) + self.remove(fn) + return False + + invalid = False + for cls in info_array[0].variants: + virtualfn = self.realfn2virtual(fn, cls) + self.clean.add(virtualfn) + if virtualfn not in self.depends_cache: + logger.debug(2, "Cache: %s is not cached", virtualfn) + invalid = True + + # If any one of the variants is not present, mark as invalid for all + if invalid: + for cls in info_array[0].variants: + virtualfn = self.realfn2virtual(fn, cls) + if virtualfn in self.clean: + logger.debug(2, "Cache: Removing %s from cache", virtualfn) + self.clean.remove(virtualfn) + if fn in self.clean: + logger.debug(2, "Cache: Marking %s as not clean", fn) + self.clean.remove(fn) + return False + + self.clean.add(fn) + return True + + def remove(self, fn): + """ + Remove a fn from the cache + Called from the parser in error cases + """ + if fn in self.depends_cache: + logger.debug(1, "Removing %s from cache", fn) + del self.depends_cache[fn] + if fn in self.clean: + logger.debug(1, "Marking %s as unclean", fn) + self.clean.remove(fn) + + def sync(self): + """ + Save the cache + Called from the parser when complete (or exiting) + """ + + if not self.has_cache: + return + + if self.cacheclean: + logger.debug(2, "Cache is clean, not saving.") + return + + file_dict = {} + pickler_dict = {} + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cache_class_name = cache_class.__name__ + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + file_dict[cache_class_name] = open(cachefile, "wb") + pickler_dict[cache_class_name] = pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL) + + pickler_dict['CoreRecipeInfo'].dump(__cache_version__) + pickler_dict['CoreRecipeInfo'].dump(bb.__version__) + + try: + for key, info_array in self.depends_cache.iteritems(): + for info in info_array: + if isinstance(info, RecipeInfoCommon): + cache_class_name = info.__class__.__name__ + pickler_dict[cache_class_name].dump(key) + pickler_dict[cache_class_name].dump(info) + finally: + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cache_class_name = cache_class.__name__ + file_dict[cache_class_name].close() + + del self.depends_cache + + @staticmethod + def mtime(cachefile): + return bb.parse.cached_mtime_noerror(cachefile) + + def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): + if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): + cacheData.add_from_recipeinfo(filename, info_array) + + if watcher: + watcher(info_array[0].file_depends) + + if not self.has_cache: + return + + if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache: + if parsed: + self.cacheclean = False + self.depends_cache[filename] = info_array + + def add(self, file_name, data, cacheData, parsed=None): + """ + Save data we need into the cache + """ + + realfn = self.virtualfn2realfn(file_name)[0] + + info_array = [] + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + info_array.append(cache_class(realfn, data)) + self.add_info(file_name, info_array, cacheData, parsed) + + @staticmethod + def load_bbfile(bbfile, appends, config): + """ + Load and parse one .bb build file + Return the data and whether parsing resulted in the file being skipped + """ + chdir_back = False + + from bb import parse + + # expand tmpdir to include this topdir + config.setVar('TMPDIR', config.getVar('TMPDIR', True) or "") + bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) + oldpath = os.path.abspath(os.getcwd()) + parse.cached_mtime_noerror(bbfile_loc) + bb_data = config.createCopy() + # The ConfHandler first looks if there is a TOPDIR and if not + # then it would call getcwd(). + # Previously, we chdir()ed to bbfile_loc, called the handler + # and finally chdir()ed back, a couple of thousand times. We now + # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet. + if not bb_data.getVar('TOPDIR', False): + chdir_back = True + bb_data.setVar('TOPDIR', bbfile_loc) + try: + if appends: + bb_data.setVar('__BBAPPEND', " ".join(appends)) + bb_data = parse.handle(bbfile, bb_data) + if chdir_back: + os.chdir(oldpath) + return bb_data + except: + if chdir_back: + os.chdir(oldpath) + raise + + +def init(cooker): + """ + The Objective: Cache the minimum amount of data possible yet get to the + stage of building packages (i.e. tryBuild) without reparsing any .bb files. + + To do this, we intercept getVar calls and only cache the variables we see + being accessed. We rely on the cache getVar calls being made for all + variables bitbake might need to use to reach this stage. For each cached + file we need to track: + + * Its mtime + * The mtimes of all its dependencies + * Whether it caused a parse.SkipRecipe exception + + Files causing parsing errors are evicted from the cache. + + """ + return Cache(cooker.configuration.data, cooker.configuration.data_hash) + + +class CacheData(object): + """ + The data structures we compile from the cached data + """ + + def __init__(self, caches_array): + self.caches_array = caches_array + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): + cache_class.init_cacheData(self) + + # Direct cache variables + self.task_queues = {} + self.preferred = {} + self.tasks = {} + # Indirect Cache variables (set elsewhere) + self.ignored_dependencies = [] + self.world_target = set() + self.bbfile_priority = {} + + def add_from_recipeinfo(self, fn, info_array): + for info in info_array: + info.add_cacheData(self, fn) + +class MultiProcessCache(object): + """ + BitBake multi-process cache implementation + + Used by the codeparser & file checksum caches + """ + + def __init__(self): + self.cachefile = None + self.cachedata = self.create_cachedata() + self.cachedata_extras = self.create_cachedata() + + def init_cache(self, d, cache_file_name=None): + cachedir = (d.getVar("PERSISTENT_DIR", True) or + d.getVar("CACHE", True)) + if cachedir in [None, '']: + return + bb.utils.mkdirhier(cachedir) + self.cachefile = os.path.join(cachedir, + cache_file_name or self.__class__.cache_file_name) + logger.debug(1, "Using cache in '%s'", self.cachefile) + + glf = bb.utils.lockfile(self.cachefile + ".lock") + + try: + with open(self.cachefile, "rb") as f: + p = pickle.Unpickler(f) + data, version = p.load() + except: + bb.utils.unlockfile(glf) + return + + bb.utils.unlockfile(glf) + + if version != self.__class__.CACHE_VERSION: + return + + self.cachedata = data + + def create_cachedata(self): + data = [{}] + return data + + def save_extras(self): + if not self.cachefile: + return + + glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True) + + i = os.getpid() + lf = None + while not lf: + lf = bb.utils.lockfile(self.cachefile + ".lock." + str(i), retry=False) + if not lf or os.path.exists(self.cachefile + "-" + str(i)): + if lf: + bb.utils.unlockfile(lf) + lf = None + i = i + 1 + continue + + with open(self.cachefile + "-" + str(i), "wb") as f: + p = pickle.Pickler(f, -1) + p.dump([self.cachedata_extras, self.__class__.CACHE_VERSION]) + + bb.utils.unlockfile(lf) + bb.utils.unlockfile(glf) + + def merge_data(self, source, dest): + for j in range(0,len(dest)): + for h in source[j]: + if h not in dest[j]: + dest[j][h] = source[j][h] + + def save_merge(self): + if not self.cachefile: + return + + glf = bb.utils.lockfile(self.cachefile + ".lock") + + data = self.cachedata + + for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]: + f = os.path.join(os.path.dirname(self.cachefile), f) + try: + with open(f, "rb") as fd: + p = pickle.Unpickler(fd) + extradata, version = p.load() + except (IOError, EOFError): + os.unlink(f) + continue + + if version != self.__class__.CACHE_VERSION: + os.unlink(f) + continue + + self.merge_data(extradata, data) + os.unlink(f) + + with open(self.cachefile, "wb") as f: + p = pickle.Pickler(f, -1) + p.dump([data, self.__class__.CACHE_VERSION]) + + bb.utils.unlockfile(glf) + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cache_extra.py b/import-layers/yocto-poky/bitbake/lib/bb/cache_extra.py new file mode 100644 index 000000000..83f4959d6 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/cache_extra.py @@ -0,0 +1,75 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Extra RecipeInfo will be all defined in this file. Currently, +# Only Hob (Image Creator) Requests some extra fields. So +# HobRecipeInfo is defined. It's named HobRecipeInfo because it +# is introduced by 'hob'. Users could also introduce other +# RecipeInfo or simply use those already defined RecipeInfo. +# In the following patch, this newly defined new extra RecipeInfo +# will be dynamically loaded and used for loading/saving the extra +# cache fields + +# Copyright (C) 2011, Intel Corporation. All rights reserved. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from bb.cache import RecipeInfoCommon + +class HobRecipeInfo(RecipeInfoCommon): + __slots__ = () + + classname = "HobRecipeInfo" + # please override this member with the correct data cache file + # such as (bb_cache.dat, bb_extracache_hob.dat) + cachefile = "bb_extracache_" + classname +".dat" + + # override this member with the list of extra cache fields + # that this class will provide + cachefields = ['summary', 'license', 'section', + 'description', 'homepage', 'bugtracker', + 'prevision', 'files_info'] + + def __init__(self, filename, metadata): + + self.summary = self.getvar('SUMMARY', metadata) + self.license = self.getvar('LICENSE', metadata) + self.section = self.getvar('SECTION', metadata) + self.description = self.getvar('DESCRIPTION', metadata) + self.homepage = self.getvar('HOMEPAGE', metadata) + self.bugtracker = self.getvar('BUGTRACKER', metadata) + self.prevision = self.getvar('PR', metadata) + self.files_info = self.getvar('FILES_INFO', metadata) + + @classmethod + def init_cacheData(cls, cachedata): + # CacheData in Hob RecipeInfo Class + cachedata.summary = {} + cachedata.license = {} + cachedata.section = {} + cachedata.description = {} + cachedata.homepage = {} + cachedata.bugtracker = {} + cachedata.prevision = {} + cachedata.files_info = {} + + def add_cacheData(self, cachedata, fn): + cachedata.summary[fn] = self.summary + cachedata.license[fn] = self.license + cachedata.section[fn] = self.section + cachedata.description[fn] = self.description + cachedata.homepage[fn] = self.homepage + cachedata.bugtracker[fn] = self.bugtracker + cachedata.prevision[fn] = self.prevision + cachedata.files_info[fn] = self.files_info diff --git a/import-layers/yocto-poky/bitbake/lib/bb/checksum.py b/import-layers/yocto-poky/bitbake/lib/bb/checksum.py new file mode 100644 index 000000000..2ec964d73 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/checksum.py @@ -0,0 +1,139 @@ +# Local file checksum cache implementation +# +# Copyright (C) 2012 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import glob +import operator +import os +import stat +import bb.utils +import logging +from bb.cache import MultiProcessCache + +logger = logging.getLogger("BitBake.Cache") + +try: + import cPickle as pickle +except ImportError: + import pickle + logger.info("Importing cPickle failed. " + "Falling back to a very slow implementation.") + + +# mtime cache (non-persistent) +# based upon the assumption that files do not change during bitbake run +class FileMtimeCache(object): + cache = {} + + def cached_mtime(self, f): + if f not in self.cache: + self.cache[f] = os.stat(f)[stat.ST_MTIME] + return self.cache[f] + + def cached_mtime_noerror(self, f): + if f not in self.cache: + try: + self.cache[f] = os.stat(f)[stat.ST_MTIME] + except OSError: + return 0 + return self.cache[f] + + def update_mtime(self, f): + self.cache[f] = os.stat(f)[stat.ST_MTIME] + return self.cache[f] + + def clear(self): + self.cache.clear() + +# Checksum + mtime cache (persistent) +class FileChecksumCache(MultiProcessCache): + cache_file_name = "local_file_checksum_cache.dat" + CACHE_VERSION = 1 + + def __init__(self): + self.mtime_cache = FileMtimeCache() + MultiProcessCache.__init__(self) + + def get_checksum(self, f): + entry = self.cachedata[0].get(f) + cmtime = self.mtime_cache.cached_mtime(f) + if entry: + (mtime, hashval) = entry + if cmtime == mtime: + return hashval + else: + bb.debug(2, "file %s changed mtime, recompute checksum" % f) + + hashval = bb.utils.md5_file(f) + self.cachedata_extras[0][f] = (cmtime, hashval) + return hashval + + def merge_data(self, source, dest): + for h in source[0]: + if h in dest: + (smtime, _) = source[0][h] + (dmtime, _) = dest[0][h] + if smtime > dmtime: + dest[0][h] = source[0][h] + else: + dest[0][h] = source[0][h] + + def get_checksums(self, filelist, pn): + """Get checksums for a list of files""" + + def checksum_file(f): + try: + checksum = self.get_checksum(f) + except OSError as e: + bb.warn("Unable to get checksum for %s SRC_URI entry %s: %s" % (pn, os.path.basename(f), e)) + return None + return checksum + + def checksum_dir(pth): + # Handle directories recursively + dirchecksums = [] + for root, dirs, files in os.walk(pth): + for name in files: + fullpth = os.path.join(root, name) + checksum = checksum_file(fullpth) + if checksum: + dirchecksums.append((fullpth, checksum)) + return dirchecksums + + checksums = [] + for pth in filelist.split(): + exist = pth.split(":")[1] + if exist == "False": + continue + pth = pth.split(":")[0] + if '*' in pth: + # Handle globs + for f in glob.glob(pth): + if os.path.isdir(f): + if not os.path.islink(f): + checksums.extend(checksum_dir(f)) + else: + checksum = checksum_file(f) + checksums.append((f, checksum)) + elif os.path.isdir(pth): + if not os.path.islink(pth): + checksums.extend(checksum_dir(pth)) + else: + checksum = checksum_file(pth) + checksums.append((pth, checksum)) + + checksums.sort(key=operator.itemgetter(1)) + return checksums diff --git a/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py b/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py new file mode 100644 index 000000000..3ee4d5622 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py @@ -0,0 +1,436 @@ +import ast +import codegen +import logging +import os.path +import bb.utils, bb.data +from itertools import chain +from pysh import pyshyacc, pyshlex, sherrors +from bb.cache import MultiProcessCache + + +logger = logging.getLogger('BitBake.CodeParser') + +try: + import cPickle as pickle +except ImportError: + import pickle + logger.info('Importing cPickle failed. Falling back to a very slow implementation.') + + +def check_indent(codestr): + """If the code is indented, add a top level piece of code to 'remove' the indentation""" + + i = 0 + while codestr[i] in ["\n", "\t", " "]: + i = i + 1 + + if i == 0: + return codestr + + if codestr[i-1] == "\t" or codestr[i-1] == " ": + if codestr[0] == "\n": + # Since we're adding a line, we need to remove one line of any empty padding + # to ensure line numbers are correct + codestr = codestr[1:] + return "if 1:\n" + codestr + + return codestr + + +# Basically pickle, in python 2.7.3 at least, does badly with data duplication +# upon pickling and unpickling. Combine this with duplicate objects and things +# are a mess. +# +# When the sets are originally created, python calls intern() on the set keys +# which significantly improves memory usage. Sadly the pickle/unpickle process +# doesn't call intern() on the keys and results in the same strings being duplicated +# in memory. This also means pickle will save the same string multiple times in +# the cache file. +# +# By having shell and python cacheline objects with setstate/getstate, we force +# the object creation through our own routine where we can call intern (via internSet). +# +# We also use hashable frozensets and ensure we use references to these so that +# duplicates can be removed, both in memory and in the resulting pickled data. +# +# By playing these games, the size of the cache file shrinks dramatically +# meaning faster load times and the reloaded cache files also consume much less +# memory. Smaller cache files, faster load times and lower memory usage is good. +# +# A custom getstate/setstate using tuples is actually worth 15% cachesize by +# avoiding duplication of the attribute names! + +class SetCache(object): + def __init__(self): + self.setcache = {} + + def internSet(self, items): + + new = [] + for i in items: + new.append(intern(i)) + s = frozenset(new) + if hash(s) in self.setcache: + return self.setcache[hash(s)] + self.setcache[hash(s)] = s + return s + +codecache = SetCache() + +class pythonCacheLine(object): + def __init__(self, refs, execs, contains): + self.refs = codecache.internSet(refs) + self.execs = codecache.internSet(execs) + self.contains = {} + for c in contains: + self.contains[c] = codecache.internSet(contains[c]) + + def __getstate__(self): + return (self.refs, self.execs, self.contains) + + def __setstate__(self, state): + (refs, execs, contains) = state + self.__init__(refs, execs, contains) + def __hash__(self): + l = (hash(self.refs), hash(self.execs)) + for c in sorted(self.contains.keys()): + l = l + (c, hash(self.contains[c])) + return hash(l) + def __repr__(self): + return " ".join([str(self.refs), str(self.execs), str(self.contains)]) + + +class shellCacheLine(object): + def __init__(self, execs): + self.execs = codecache.internSet(execs) + + def __getstate__(self): + return (self.execs) + + def __setstate__(self, state): + (execs) = state + self.__init__(execs) + def __hash__(self): + return hash(self.execs) + def __repr__(self): + return str(self.execs) + +class CodeParserCache(MultiProcessCache): + cache_file_name = "bb_codeparser.dat" + CACHE_VERSION = 7 + + def __init__(self): + MultiProcessCache.__init__(self) + self.pythoncache = self.cachedata[0] + self.shellcache = self.cachedata[1] + self.pythoncacheextras = self.cachedata_extras[0] + self.shellcacheextras = self.cachedata_extras[1] + + # To avoid duplication in the codeparser cache, keep + # a lookup of hashes of objects we already have + self.pythoncachelines = {} + self.shellcachelines = {} + + def newPythonCacheLine(self, refs, execs, contains): + cacheline = pythonCacheLine(refs, execs, contains) + h = hash(cacheline) + if h in self.pythoncachelines: + return self.pythoncachelines[h] + self.pythoncachelines[h] = cacheline + return cacheline + + def newShellCacheLine(self, execs): + cacheline = shellCacheLine(execs) + h = hash(cacheline) + if h in self.shellcachelines: + return self.shellcachelines[h] + self.shellcachelines[h] = cacheline + return cacheline + + def init_cache(self, d): + # Check if we already have the caches + if self.pythoncache: + return + + MultiProcessCache.init_cache(self, d) + + # cachedata gets re-assigned in the parent + self.pythoncache = self.cachedata[0] + self.shellcache = self.cachedata[1] + + def create_cachedata(self): + data = [{}, {}] + return data + +codeparsercache = CodeParserCache() + +def parser_cache_init(d): + codeparsercache.init_cache(d) + +def parser_cache_save(): + codeparsercache.save_extras() + +def parser_cache_savemerge(): + codeparsercache.save_merge() + +Logger = logging.getLoggerClass() +class BufferedLogger(Logger): + def __init__(self, name, level=0, target=None): + Logger.__init__(self, name) + self.setLevel(level) + self.buffer = [] + self.target = target + + def handle(self, record): + self.buffer.append(record) + + def flush(self): + for record in self.buffer: + self.target.handle(record) + self.buffer = [] + +class PythonParser(): + getvars = (".getVar", ".appendVar", ".prependVar") + containsfuncs = ("bb.utils.contains", "base_contains", "bb.utils.contains_any") + execfuncs = ("bb.build.exec_func", "bb.build.exec_task") + + def warn(self, func, arg): + """Warn about calls of bitbake APIs which pass a non-literal + argument for the variable name, as we're not able to track such + a reference. + """ + + try: + funcstr = codegen.to_source(func) + argstr = codegen.to_source(arg) + except TypeError: + self.log.debug(2, 'Failed to convert function and argument to source form') + else: + self.log.debug(1, self.unhandled_message % (funcstr, argstr)) + + def visit_Call(self, node): + name = self.called_node_name(node.func) + if name and name.endswith(self.getvars) or name in self.containsfuncs: + if isinstance(node.args[0], ast.Str): + varname = node.args[0].s + if name in self.containsfuncs and isinstance(node.args[1], ast.Str): + if varname not in self.contains: + self.contains[varname] = set() + self.contains[varname].add(node.args[1].s) + else: + self.references.add(node.args[0].s) + else: + self.warn(node.func, node.args[0]) + elif name and name.endswith(".expand"): + if isinstance(node.args[0], ast.Str): + value = node.args[0].s + d = bb.data.init() + parser = d.expandWithRefs(value, self.name) + self.references |= parser.references + self.execs |= parser.execs + for varname in parser.contains: + if varname not in self.contains: + self.contains[varname] = set() + self.contains[varname] |= parser.contains[varname] + elif name in self.execfuncs: + if isinstance(node.args[0], ast.Str): + self.var_execs.add(node.args[0].s) + else: + self.warn(node.func, node.args[0]) + elif name and isinstance(node.func, (ast.Name, ast.Attribute)): + self.execs.add(name) + + def called_node_name(self, node): + """Given a called node, return its original string form""" + components = [] + while node: + if isinstance(node, ast.Attribute): + components.append(node.attr) + node = node.value + elif isinstance(node, ast.Name): + components.append(node.id) + return '.'.join(reversed(components)) + else: + break + + def __init__(self, name, log): + self.name = name + self.var_execs = set() + self.contains = {} + self.execs = set() + self.references = set() + self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, log) + + self.unhandled_message = "in call of %s, argument '%s' is not a string literal" + self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message) + + def parse_python(self, node, lineno=0, filename="<string>"): + if not node or not node.strip(): + return + + h = hash(str(node)) + + if h in codeparsercache.pythoncache: + self.references = set(codeparsercache.pythoncache[h].refs) + self.execs = set(codeparsercache.pythoncache[h].execs) + self.contains = {} + for i in codeparsercache.pythoncache[h].contains: + self.contains[i] = set(codeparsercache.pythoncache[h].contains[i]) + return + + if h in codeparsercache.pythoncacheextras: + self.references = set(codeparsercache.pythoncacheextras[h].refs) + self.execs = set(codeparsercache.pythoncacheextras[h].execs) + self.contains = {} + for i in codeparsercache.pythoncacheextras[h].contains: + self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i]) + return + + # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though + node = "\n" * int(lineno) + node + code = compile(check_indent(str(node)), filename, "exec", + ast.PyCF_ONLY_AST) + + for n in ast.walk(code): + if n.__class__.__name__ == "Call": + self.visit_Call(n) + + self.execs.update(self.var_execs) + + codeparsercache.pythoncacheextras[h] = codeparsercache.newPythonCacheLine(self.references, self.execs, self.contains) + +class ShellParser(): + def __init__(self, name, log): + self.funcdefs = set() + self.allexecs = set() + self.execs = set() + self.log = BufferedLogger('BitBake.Data.%s' % name, logging.DEBUG, log) + self.unhandled_template = "unable to handle non-literal command '%s'" + self.unhandled_template = "while parsing %s, %s" % (name, self.unhandled_template) + + def parse_shell(self, value): + """Parse the supplied shell code in a string, returning the external + commands it executes. + """ + + h = hash(str(value)) + + if h in codeparsercache.shellcache: + self.execs = set(codeparsercache.shellcache[h].execs) + return self.execs + + if h in codeparsercache.shellcacheextras: + self.execs = set(codeparsercache.shellcacheextras[h].execs) + return self.execs + + self._parse_shell(value) + self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs) + + codeparsercache.shellcacheextras[h] = codeparsercache.newShellCacheLine(self.execs) + + return self.execs + + def _parse_shell(self, value): + try: + tokens, _ = pyshyacc.parse(value, eof=True, debug=False) + except pyshlex.NeedMore: + raise sherrors.ShellSyntaxError("Unexpected EOF") + + for token in tokens: + self.process_tokens(token) + + def process_tokens(self, tokens): + """Process a supplied portion of the syntax tree as returned by + pyshyacc.parse. + """ + + def function_definition(value): + self.funcdefs.add(value.name) + return [value.body], None + + def case_clause(value): + # Element 0 of each item in the case is the list of patterns, and + # Element 1 of each item in the case is the list of commands to be + # executed when that pattern matches. + words = chain(*[item[0] for item in value.items]) + cmds = chain(*[item[1] for item in value.items]) + return cmds, words + + def if_clause(value): + main = chain(value.cond, value.if_cmds) + rest = value.else_cmds + if isinstance(rest, tuple) and rest[0] == "elif": + return chain(main, if_clause(rest[1])) + else: + return chain(main, rest) + + def simple_command(value): + return None, chain(value.words, (assign[1] for assign in value.assigns)) + + token_handlers = { + "and_or": lambda x: ((x.left, x.right), None), + "async": lambda x: ([x], None), + "brace_group": lambda x: (x.cmds, None), + "for_clause": lambda x: (x.cmds, x.items), + "function_definition": function_definition, + "if_clause": lambda x: (if_clause(x), None), + "pipeline": lambda x: (x.commands, None), + "redirect_list": lambda x: ([x.cmd], None), + "subshell": lambda x: (x.cmds, None), + "while_clause": lambda x: (chain(x.condition, x.cmds), None), + "until_clause": lambda x: (chain(x.condition, x.cmds), None), + "simple_command": simple_command, + "case_clause": case_clause, + } + + for token in tokens: + name, value = token + try: + more_tokens, words = token_handlers[name](value) + except KeyError: + raise NotImplementedError("Unsupported token type " + name) + + if more_tokens: + self.process_tokens(more_tokens) + + if words: + self.process_words(words) + + def process_words(self, words): + """Process a set of 'words' in pyshyacc parlance, which includes + extraction of executed commands from $() blocks, as well as grabbing + the command name argument. + """ + + words = list(words) + for word in list(words): + wtree = pyshlex.make_wordtree(word[1]) + for part in wtree: + if not isinstance(part, list): + continue + + if part[0] in ('`', '$('): + command = pyshlex.wordtree_as_string(part[1:-1]) + self._parse_shell(command) + + if word[0] in ("cmd_name", "cmd_word"): + if word in words: + words.remove(word) + + usetoken = False + for word in words: + if word[0] in ("cmd_name", "cmd_word") or \ + (usetoken and word[0] == "TOKEN"): + if "=" in word[1]: + usetoken = True + continue + + cmd = word[1] + if cmd.startswith("$"): + self.log.debug(1, self.unhandled_template % cmd) + elif cmd == "eval": + command = " ".join(word for _, word in words[1:]) + self._parse_shell(command) + else: + self.allexecs.add(cmd) + break diff --git a/import-layers/yocto-poky/bitbake/lib/bb/command.py b/import-layers/yocto-poky/bitbake/lib/bb/command.py new file mode 100644 index 000000000..0559ffc07 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/command.py @@ -0,0 +1,474 @@ +""" +BitBake 'Command' module + +Provide an interface to interact with the bitbake server through 'commands' +""" + +# Copyright (C) 2006-2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +The bitbake server takes 'commands' from its UI/commandline. +Commands are either synchronous or asynchronous. +Async commands return data to the client in the form of events. +Sync commands must only return data through the function return value +and must not trigger events, directly or indirectly. +Commands are queued in a CommandQueue +""" + +import bb.event +import bb.cooker + +class CommandCompleted(bb.event.Event): + pass + +class CommandExit(bb.event.Event): + def __init__(self, exitcode): + bb.event.Event.__init__(self) + self.exitcode = int(exitcode) + +class CommandFailed(CommandExit): + def __init__(self, message): + self.error = message + CommandExit.__init__(self, 1) + +class CommandError(Exception): + pass + +class Command: + """ + A queue of asynchronous commands for bitbake + """ + def __init__(self, cooker): + self.cooker = cooker + self.cmds_sync = CommandsSync() + self.cmds_async = CommandsAsync() + + # FIXME Add lock for this + self.currentAsyncCommand = None + + def runCommand(self, commandline, ro_only = False): + command = commandline.pop(0) + if hasattr(CommandsSync, command): + # Can run synchronous commands straight away + command_method = getattr(self.cmds_sync, command) + if ro_only: + if not hasattr(command_method, 'readonly') or False == getattr(command_method, 'readonly'): + return None, "Not able to execute not readonly commands in readonly mode" + try: + if getattr(command_method, 'needconfig', False): + self.cooker.updateCacheSync() + result = command_method(self, commandline) + except CommandError as exc: + return None, exc.args[0] + except (Exception, SystemExit): + import traceback + return None, traceback.format_exc() + else: + return result, None + if self.currentAsyncCommand is not None: + return None, "Busy (%s in progress)" % self.currentAsyncCommand[0] + if command not in CommandsAsync.__dict__: + return None, "No such command" + self.currentAsyncCommand = (command, commandline) + self.cooker.configuration.server_register_idlecallback(self.cooker.runCommands, self.cooker) + return True, None + + def runAsyncCommand(self): + try: + if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown): + # updateCache will trigger a shutdown of the parser + # and then raise BBHandledException triggering an exit + self.cooker.updateCache() + return False + if self.currentAsyncCommand is not None: + (command, options) = self.currentAsyncCommand + commandmethod = getattr(CommandsAsync, command) + needcache = getattr( commandmethod, "needcache" ) + if needcache and self.cooker.state != bb.cooker.state.running: + self.cooker.updateCache() + return True + else: + commandmethod(self.cmds_async, self, options) + return False + else: + return False + except KeyboardInterrupt as exc: + self.finishAsyncCommand("Interrupted") + return False + except SystemExit as exc: + arg = exc.args[0] + if isinstance(arg, basestring): + self.finishAsyncCommand(arg) + else: + self.finishAsyncCommand("Exited with %s" % arg) + return False + except Exception as exc: + import traceback + if isinstance(exc, bb.BBHandledException): + self.finishAsyncCommand("") + else: + self.finishAsyncCommand(traceback.format_exc()) + return False + + def finishAsyncCommand(self, msg=None, code=None): + if msg or msg == "": + bb.event.fire(CommandFailed(msg), self.cooker.expanded_data) + elif code: + bb.event.fire(CommandExit(code), self.cooker.expanded_data) + else: + bb.event.fire(CommandCompleted(), self.cooker.expanded_data) + self.currentAsyncCommand = None + self.cooker.finishcommand() + +class CommandsSync: + """ + A class of synchronous commands + These should run quickly so as not to hurt interactive performance. + These must not influence any running synchronous command. + """ + + def stateShutdown(self, command, params): + """ + Trigger cooker 'shutdown' mode + """ + command.cooker.shutdown(False) + + def stateForceShutdown(self, command, params): + """ + Stop the cooker + """ + command.cooker.shutdown(True) + + def getAllKeysWithFlags(self, command, params): + """ + Returns a dump of the global state. Call with + variable flags to be retrieved as params. + """ + flaglist = params[0] + return command.cooker.getAllKeysWithFlags(flaglist) + getAllKeysWithFlags.readonly = True + + def getVariable(self, command, params): + """ + Read the value of a variable from data + """ + varname = params[0] + expand = True + if len(params) > 1: + expand = (params[1] == "True") + + return command.cooker.data.getVar(varname, expand) + getVariable.readonly = True + + def setVariable(self, command, params): + """ + Set the value of variable in data + """ + varname = params[0] + value = str(params[1]) + command.cooker.data.setVar(varname, value) + + def getSetVariable(self, command, params): + """ + Read the value of a variable from data and set it into the datastore + which effectively expands and locks the value. + """ + varname = params[0] + result = self.getVariable(command, params) + command.cooker.data.setVar(varname, result) + return result + + def setConfig(self, command, params): + """ + Set the value of variable in configuration + """ + varname = params[0] + value = str(params[1]) + setattr(command.cooker.configuration, varname, value) + + def enableDataTracking(self, command, params): + """ + Enable history tracking for variables + """ + command.cooker.enableDataTracking() + + def disableDataTracking(self, command, params): + """ + Disable history tracking for variables + """ + command.cooker.disableDataTracking() + + def setPrePostConfFiles(self, command, params): + prefiles = params[0].split() + postfiles = params[1].split() + command.cooker.configuration.prefile = prefiles + command.cooker.configuration.postfile = postfiles + setPrePostConfFiles.needconfig = False + + def getCpuCount(self, command, params): + """ + Get the CPU count on the bitbake server + """ + return bb.utils.cpu_count() + getCpuCount.readonly = True + getCpuCount.needconfig = False + + def matchFile(self, command, params): + fMatch = params[0] + return command.cooker.matchFile(fMatch) + matchFile.needconfig = False + + def generateNewImage(self, command, params): + image = params[0] + base_image = params[1] + package_queue = params[2] + timestamp = params[3] + description = params[4] + return command.cooker.generateNewImage(image, base_image, + package_queue, timestamp, description) + + def ensureDir(self, command, params): + directory = params[0] + bb.utils.mkdirhier(directory) + ensureDir.needconfig = False + + def setVarFile(self, command, params): + """ + Save a variable in a file; used for saving in a configuration file + """ + var = params[0] + val = params[1] + default_file = params[2] + op = params[3] + command.cooker.modifyConfigurationVar(var, val, default_file, op) + setVarFile.needconfig = False + + def removeVarFile(self, command, params): + """ + Remove a variable declaration from a file + """ + var = params[0] + command.cooker.removeConfigurationVar(var) + removeVarFile.needconfig = False + + def createConfigFile(self, command, params): + """ + Create an extra configuration file + """ + name = params[0] + command.cooker.createConfigFile(name) + createConfigFile.needconfig = False + + def setEventMask(self, command, params): + handlerNum = params[0] + llevel = params[1] + debug_domains = params[2] + mask = params[3] + return bb.event.set_UIHmask(handlerNum, llevel, debug_domains, mask) + setEventMask.needconfig = False + setEventMask.readonly = True + + def setFeatures(self, command, params): + """ + Set the cooker features to include the passed list of features + """ + features = params[0] + command.cooker.setFeatures(features) + setFeatures.needconfig = False + # although we change the internal state of the cooker, this is transparent since + # we always take and leave the cooker in state.initial + setFeatures.readonly = True + + def updateConfig(self, command, params): + options = params[0] + environment = params[1] + command.cooker.updateConfigOpts(options, environment) + updateConfig.needconfig = False + +class CommandsAsync: + """ + A class of asynchronous commands + These functions communicate via generated events. + Any function that requires metadata parsing should be here. + """ + + def buildFile(self, command, params): + """ + Build a single specified .bb file + """ + bfile = params[0] + task = params[1] + + command.cooker.buildFile(bfile, task) + buildFile.needcache = False + + def buildTargets(self, command, params): + """ + Build a set of targets + """ + pkgs_to_build = params[0] + task = params[1] + + command.cooker.buildTargets(pkgs_to_build, task) + buildTargets.needcache = True + + def generateDepTreeEvent(self, command, params): + """ + Generate an event containing the dependency information + """ + pkgs_to_build = params[0] + task = params[1] + + command.cooker.generateDepTreeEvent(pkgs_to_build, task) + command.finishAsyncCommand() + generateDepTreeEvent.needcache = True + + def generateDotGraph(self, command, params): + """ + Dump dependency information to disk as .dot files + """ + pkgs_to_build = params[0] + task = params[1] + + command.cooker.generateDotGraphFiles(pkgs_to_build, task) + command.finishAsyncCommand() + generateDotGraph.needcache = True + + def generateTargetsTree(self, command, params): + """ + Generate a tree of buildable targets. + If klass is provided ensure all recipes that inherit the class are + included in the package list. + If pkg_list provided use that list (plus any extras brought in by + klass) rather than generating a tree for all packages. + """ + klass = params[0] + pkg_list = params[1] + + command.cooker.generateTargetsTree(klass, pkg_list) + command.finishAsyncCommand() + generateTargetsTree.needcache = True + + def findCoreBaseFiles(self, command, params): + """ + Find certain files in COREBASE directory. i.e. Layers + """ + subdir = params[0] + filename = params[1] + + command.cooker.findCoreBaseFiles(subdir, filename) + command.finishAsyncCommand() + findCoreBaseFiles.needcache = False + + def findConfigFiles(self, command, params): + """ + Find config files which provide appropriate values + for the passed configuration variable. i.e. MACHINE + """ + varname = params[0] + + command.cooker.findConfigFiles(varname) + command.finishAsyncCommand() + findConfigFiles.needcache = False + + def findFilesMatchingInDir(self, command, params): + """ + Find implementation files matching the specified pattern + in the requested subdirectory of a BBPATH + """ + pattern = params[0] + directory = params[1] + + command.cooker.findFilesMatchingInDir(pattern, directory) + command.finishAsyncCommand() + findFilesMatchingInDir.needcache = False + + def findConfigFilePath(self, command, params): + """ + Find the path of the requested configuration file + """ + configfile = params[0] + + command.cooker.findConfigFilePath(configfile) + command.finishAsyncCommand() + findConfigFilePath.needcache = False + + def showVersions(self, command, params): + """ + Show the currently selected versions + """ + command.cooker.showVersions() + command.finishAsyncCommand() + showVersions.needcache = True + + def showEnvironmentTarget(self, command, params): + """ + Print the environment of a target recipe + (needs the cache to work out which recipe to use) + """ + pkg = params[0] + + command.cooker.showEnvironment(None, pkg) + command.finishAsyncCommand() + showEnvironmentTarget.needcache = True + + def showEnvironment(self, command, params): + """ + Print the standard environment + or if specified the environment for a specified recipe + """ + bfile = params[0] + + command.cooker.showEnvironment(bfile) + command.finishAsyncCommand() + showEnvironment.needcache = False + + def parseFiles(self, command, params): + """ + Parse the .bb files + """ + command.cooker.updateCache() + command.finishAsyncCommand() + parseFiles.needcache = True + + def compareRevisions(self, command, params): + """ + Parse the .bb files + """ + if bb.fetch.fetcher_compare_revisions(command.cooker.data): + command.finishAsyncCommand(code=1) + else: + command.finishAsyncCommand() + compareRevisions.needcache = True + + def triggerEvent(self, command, params): + """ + Trigger a certain event + """ + event = params[0] + bb.event.fire(eval(event), command.cooker.data) + command.currentAsyncCommand = None + triggerEvent.needcache = False + + def resetCooker(self, command, params): + """ + Reset the cooker to its initial state, thus forcing a reparse for + any async command that has the needcache property set to True + """ + command.cooker.reset() + command.finishAsyncCommand() + resetCooker.needcache = False + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/compat.py b/import-layers/yocto-poky/bitbake/lib/bb/compat.py new file mode 100644 index 000000000..de1923d28 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/compat.py @@ -0,0 +1,6 @@ +"""Code pulled from future python versions, here for compatibility""" + +from collections import MutableMapping, KeysView, ValuesView, ItemsView, OrderedDict +from functools import total_ordering + + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cooker.py b/import-layers/yocto-poky/bitbake/lib/bb/cooker.py new file mode 100644 index 000000000..9b565fc37 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/cooker.py @@ -0,0 +1,2196 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH +# Copyright (C) 2006 - 2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import print_function +import sys, os, glob, os.path, re, time +import atexit +import itertools +import logging +import multiprocessing +import sre_constants +import threading +from cStringIO import StringIO +from contextlib import closing +from functools import wraps +from collections import defaultdict +import bb, bb.exceptions, bb.command +from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build +import Queue +import signal +import subprocess +import errno +import prserv.serv +import pyinotify + +logger = logging.getLogger("BitBake") +collectlog = logging.getLogger("BitBake.Collection") +buildlog = logging.getLogger("BitBake.Build") +parselog = logging.getLogger("BitBake.Parsing") +providerlog = logging.getLogger("BitBake.Provider") + +class NoSpecificMatch(bb.BBHandledException): + """ + Exception raised when no or multiple file matches are found + """ + +class NothingToBuild(Exception): + """ + Exception raised when there is nothing to build + """ + +class CollectionError(bb.BBHandledException): + """ + Exception raised when layer configuration is incorrect + """ + +class state: + initial, parsing, running, shutdown, forceshutdown, stopped, error = range(7) + + @classmethod + def get_name(cls, code): + for name in dir(cls): + value = getattr(cls, name) + if type(value) == type(cls.initial) and value == code: + return name + raise ValueError("Invalid status code: %s" % code) + + +class SkippedPackage: + def __init__(self, info = None, reason = None): + self.pn = None + self.skipreason = None + self.provides = None + self.rprovides = None + + if info: + self.pn = info.pn + self.skipreason = info.skipreason + self.provides = info.provides + self.rprovides = info.rprovides + elif reason: + self.skipreason = reason + + +class CookerFeatures(object): + _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS] = range(4) + + def __init__(self): + self._features=set() + + def setFeature(self, f): + # validate we got a request for a feature we support + if f not in CookerFeatures._feature_list: + return + self._features.add(f) + + def __contains__(self, f): + return f in self._features + + def __iter__(self): + return self._features.__iter__() + + def next(self): + return self._features.next() + + +#============================================================================# +# BBCooker +#============================================================================# +class BBCooker: + """ + Manages one bitbake build run + """ + + def __init__(self, configuration, featureSet=None): + self.recipecache = None + self.skiplist = {} + self.featureset = CookerFeatures() + if featureSet: + for f in featureSet: + self.featureset.setFeature(f) + + self.configuration = configuration + + self.configwatcher = pyinotify.WatchManager() + self.configwatcher.bbseen = [] + self.configwatcher.bbwatchedfiles = [] + self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications) + self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \ + pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \ + pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO + self.watcher = pyinotify.WatchManager() + self.watcher.bbseen = [] + self.watcher.bbwatchedfiles = [] + self.notifier = pyinotify.Notifier(self.watcher, self.notifications) + + # If being called by something like tinfoil, we need to clean cached data + # which may now be invalid + bb.parse.__mtime_cache = {} + bb.parse.BBHandler.cached_statements = {} + + self.initConfigurationData() + + self.inotify_modified_files = [] + + def _process_inotify_updates(server, notifier_list, abort): + for n in notifier_list: + if n.check_events(timeout=0): + # read notified events and enqeue them + n.read_events() + n.process_events() + return 1.0 + + self.configuration.server_register_idlecallback(_process_inotify_updates, [self.confignotifier, self.notifier]) + + self.baseconfig_valid = True + self.parsecache_valid = False + + # Take a lock so only one copy of bitbake can run against a given build + # directory at a time + if not self.lockBitbake(): + bb.fatal("Only one copy of bitbake should be run against a build directory") + try: + self.lock.seek(0) + self.lock.truncate() + if len(configuration.interface) >= 2: + self.lock.write("%s:%s\n" % (configuration.interface[0], configuration.interface[1])); + self.lock.flush() + except: + pass + + # TOSTOP must not be set or our children will hang when they output + fd = sys.stdout.fileno() + if os.isatty(fd): + import termios + tcattr = termios.tcgetattr(fd) + if tcattr[3] & termios.TOSTOP: + buildlog.info("The terminal had the TOSTOP bit set, clearing...") + tcattr[3] = tcattr[3] & ~termios.TOSTOP + termios.tcsetattr(fd, termios.TCSANOW, tcattr) + + self.command = bb.command.Command(self) + self.state = state.initial + + self.parser = None + + signal.signal(signal.SIGTERM, self.sigterm_exception) + # Let SIGHUP exit as SIGTERM + signal.signal(signal.SIGHUP, self.sigterm_exception) + + def config_notifications(self, event): + if not event.pathname in self.configwatcher.bbwatchedfiles: + return + if not event.pathname in self.inotify_modified_files: + self.inotify_modified_files.append(event.pathname) + self.baseconfig_valid = False + + def notifications(self, event): + if not event.pathname in self.inotify_modified_files: + self.inotify_modified_files.append(event.pathname) + self.parsecache_valid = False + + def add_filewatch(self, deps, watcher=None): + if not watcher: + watcher = self.watcher + for i in deps: + watcher.bbwatchedfiles.append(i[0]) + f = os.path.dirname(i[0]) + if f in watcher.bbseen: + continue + watcher.bbseen.append(f) + watchtarget = None + while True: + # We try and add watches for files that don't exist but if they did, would influence + # the parser. The parent directory of these files may not exist, in which case we need + # to watch any parent that does exist for changes. + try: + watcher.add_watch(f, self.watchmask, quiet=False) + if watchtarget: + watcher.bbwatchedfiles.append(watchtarget) + break + except pyinotify.WatchManagerError as e: + if 'ENOENT' in str(e): + watchtarget = f + f = os.path.dirname(f) + if f in watcher.bbseen: + break + watcher.bbseen.append(f) + continue + if 'ENOSPC' in str(e): + providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?") + providerlog.error("To check max_user_watches: sysctl -n fs.inotify.max_user_watches.") + providerlog.error("To modify max_user_watches: sysctl -n -w fs.inotify.max_user_watches=<value>.") + providerlog.error("Root privilege is required to modify max_user_watches.") + raise + + def sigterm_exception(self, signum, stackframe): + if signum == signal.SIGTERM: + bb.warn("Cooker received SIGTERM, shutting down...") + elif signum == signal.SIGHUP: + bb.warn("Cooker received SIGHUP, shutting down...") + self.state = state.forceshutdown + + def setFeatures(self, features): + # we only accept a new feature set if we're in state initial, so we can reset without problems + if not self.state in [state.initial, state.shutdown, state.forceshutdown, state.stopped, state.error]: + raise Exception("Illegal state for feature set change") + original_featureset = list(self.featureset) + for feature in features: + self.featureset.setFeature(feature) + bb.debug(1, "Features set %s (was %s)" % (original_featureset, list(self.featureset))) + if (original_featureset != list(self.featureset)) and self.state != state.error: + self.reset() + + def initConfigurationData(self): + + self.state = state.initial + self.caches_array = [] + + # Need to preserve BB_CONSOLELOG over resets + consolelog = None + if hasattr(self, "data"): + consolelog = self.data.getVar("BB_CONSOLELOG", True) + + if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset: + self.enableDataTracking() + + all_extra_cache_names = [] + # We hardcode all known cache types in a single place, here. + if CookerFeatures.HOB_EXTRA_CACHES in self.featureset: + all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo") + + caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names + + # At least CoreRecipeInfo will be loaded, so caches_array will never be empty! + # This is the entry point, no further check needed! + for var in caches_name_array: + try: + module_name, cache_name = var.split(':') + module = __import__(module_name, fromlist=(cache_name,)) + self.caches_array.append(getattr(module, cache_name)) + except ImportError as exc: + logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc)) + sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name) + + self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False) + self.databuilder.parseBaseConfiguration() + self.data = self.databuilder.data + self.data_hash = self.databuilder.data_hash + + if consolelog: + self.data.setVar("BB_CONSOLELOG", consolelog) + + # we log all events to a file if so directed + if self.configuration.writeeventlog: + import json, pickle + DEFAULT_EVENTFILE = self.configuration.writeeventlog + class EventLogWriteHandler(): + + class EventWriter(): + def __init__(self, cooker): + self.file_inited = None + self.cooker = cooker + self.event_queue = [] + + def init_file(self): + try: + # delete the old log + os.remove(DEFAULT_EVENTFILE) + except: + pass + + # write current configuration data + with open(DEFAULT_EVENTFILE, "w") as f: + f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) + + def write_event(self, event): + with open(DEFAULT_EVENTFILE, "a") as f: + try: + f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) })) + except Exception as e: + import traceback + print(e, traceback.format_exc(e)) + + + def send(self, event): + event_class = event.__module__ + "." + event.__class__.__name__ + + # init on bb.event.BuildStarted + if self.file_inited is None: + if event_class == "bb.event.BuildStarted": + self.init_file() + self.file_inited = True + + # write pending events + for e in self.event_queue: + self.write_event(e) + + # also write the current event + self.write_event(event) + + else: + # queue all events until the file is inited + self.event_queue.append(event) + + else: + # we have the file, just write the event + self.write_event(event) + + # set our handler's event processor + event = EventWriter(self) # self is the cooker here + + + # set up cooker features for this mock UI handler + + # we need to write the dependency tree in the log + self.featureset.setFeature(CookerFeatures.SEND_DEPENDS_TREE) + # register the log file writer as UI Handler + bb.event.register_UIHhandler(EventLogWriteHandler()) + + + # + # Copy of the data store which has been expanded. + # Used for firing events and accessing variables where expansion needs to be accounted for + # + self.expanded_data = bb.data.createCopy(self.data) + bb.data.update_data(self.expanded_data) + bb.parse.init_parser(self.expanded_data) + + if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset: + self.disableDataTracking() + + self.data.renameVar("__depends", "__base_depends") + self.add_filewatch(self.data.getVar("__base_depends", False), self.configwatcher) + + + def enableDataTracking(self): + self.configuration.tracking = True + if hasattr(self, "data"): + self.data.enableTracking() + + def disableDataTracking(self): + self.configuration.tracking = False + if hasattr(self, "data"): + self.data.disableTracking() + + def modifyConfigurationVar(self, var, val, default_file, op): + if op == "append": + self.appendConfigurationVar(var, val, default_file) + elif op == "set": + self.saveConfigurationVar(var, val, default_file, "=") + elif op == "earlyAssign": + self.saveConfigurationVar(var, val, default_file, "?=") + + + def appendConfigurationVar(self, var, val, default_file): + #add append var operation to the end of default_file + default_file = bb.cookerdata.findConfigFile(default_file, self.data) + + total = "#added by hob" + total += "\n%s += \"%s\"\n" % (var, val) + + with open(default_file, 'a') as f: + f.write(total) + + #add to history + loginfo = {"op":"append", "file":default_file, "line":total.count("\n")} + self.data.appendVar(var, val, **loginfo) + + def saveConfigurationVar(self, var, val, default_file, op): + + replaced = False + #do not save if nothing changed + if str(val) == self.data.getVar(var, False): + return + + conf_files = self.data.varhistory.get_variable_files(var) + + #format the value when it is a list + if isinstance(val, list): + listval = "" + for value in val: + listval += "%s " % value + val = listval + + topdir = self.data.getVar("TOPDIR", False) + + #comment or replace operations made on var + for conf_file in conf_files: + if topdir in conf_file: + with open(conf_file, 'r') as f: + contents = f.readlines() + + lines = self.data.varhistory.get_variable_lines(var, conf_file) + for line in lines: + total = "" + i = 0 + for c in contents: + total += c + i = i + 1 + if i==int(line): + end_index = len(total) + index = total.rfind(var, 0, end_index) + + begin_line = total.count("\n",0,index) + end_line = int(line) + + #check if the variable was saved before in the same way + #if true it replace the place where the variable was declared + #else it comments it + if contents[begin_line-1]== "#added by hob\n": + contents[begin_line] = "%s %s \"%s\"\n" % (var, op, val) + replaced = True + else: + for ii in range(begin_line, end_line): + contents[ii] = "#" + contents[ii] + + with open(conf_file, 'w') as f: + f.writelines(contents) + + if replaced == False: + #remove var from history + self.data.varhistory.del_var_history(var) + + #add var to the end of default_file + default_file = bb.cookerdata.findConfigFile(default_file, self.data) + + #add the variable on a single line, to be easy to replace the second time + total = "\n#added by hob" + total += "\n%s %s \"%s\"\n" % (var, op, val) + + with open(default_file, 'a') as f: + f.write(total) + + #add to history + loginfo = {"op":"set", "file":default_file, "line":total.count("\n")} + self.data.setVar(var, val, **loginfo) + + def removeConfigurationVar(self, var): + conf_files = self.data.varhistory.get_variable_files(var) + topdir = self.data.getVar("TOPDIR", False) + + for conf_file in conf_files: + if topdir in conf_file: + with open(conf_file, 'r') as f: + contents = f.readlines() + + lines = self.data.varhistory.get_variable_lines(var, conf_file) + for line in lines: + total = "" + i = 0 + for c in contents: + total += c + i = i + 1 + if i==int(line): + end_index = len(total) + index = total.rfind(var, 0, end_index) + + begin_line = total.count("\n",0,index) + + #check if the variable was saved before in the same way + if contents[begin_line-1]== "#added by hob\n": + contents[begin_line-1] = contents[begin_line] = "\n" + else: + contents[begin_line] = "\n" + #remove var from history + self.data.varhistory.del_var_history(var, conf_file, line) + #remove variable + self.data.delVar(var) + + with open(conf_file, 'w') as f: + f.writelines(contents) + + def createConfigFile(self, name): + path = os.getcwd() + confpath = os.path.join(path, "conf", name) + open(confpath, 'w').close() + + def parseConfiguration(self): + # Set log file verbosity + verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", False)) + if verboselogs: + bb.msg.loggerVerboseLogs = True + + # Change nice level if we're asked to + nice = self.data.getVar("BB_NICE_LEVEL", True) + if nice: + curnice = os.nice(0) + nice = int(nice) - curnice + buildlog.verbose("Renice to %s " % os.nice(nice)) + + if self.recipecache: + del self.recipecache + self.recipecache = bb.cache.CacheData(self.caches_array) + + self.handleCollections( self.data.getVar("BBFILE_COLLECTIONS", True) ) + + def updateConfigOpts(self, options, environment): + clean = True + for o in options: + if o in ['prefile', 'postfile']: + clean = False + server_val = getattr(self.configuration, "%s_server" % o) + if not options[o] and server_val: + # restore value provided on server start + setattr(self.configuration, o, server_val) + continue + setattr(self.configuration, o, options[o]) + for k in bb.utils.approved_variables(): + if k in environment and k not in self.configuration.env: + logger.debug(1, "Updating environment variable %s to %s" % (k, environment[k])) + self.configuration.env[k] = environment[k] + clean = False + if k in self.configuration.env and k not in environment: + logger.debug(1, "Updating environment variable %s (deleted)" % (k)) + del self.configuration.env[k] + clean = False + if k not in self.configuration.env and k not in environment: + continue + if environment[k] != self.configuration.env[k]: + logger.debug(1, "Updating environment variable %s to %s" % (k, environment[k])) + self.configuration.env[k] = environment[k] + clean = False + if not clean: + logger.debug(1, "Base environment change, triggering reparse") + self.baseconfig_valid = False + self.reset() + + def runCommands(self, server, data, abort): + """ + Run any queued asynchronous command + This is done by the idle handler so it runs in true context rather than + tied to any UI. + """ + + return self.command.runAsyncCommand() + + def showVersions(self): + + pkg_pn = self.recipecache.pkg_pn + (latest_versions, preferred_versions) = bb.providers.findProviders(self.data, self.recipecache, pkg_pn) + + logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version") + logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================") + + for p in sorted(pkg_pn): + pref = preferred_versions[p] + latest = latest_versions[p] + + prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2] + lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2] + + if pref == latest: + prefstr = "" + + logger.plain("%-35s %25s %25s", p, lateststr, prefstr) + + def showEnvironment(self, buildfile=None, pkgs_to_build=None): + """ + Show the outer or per-recipe environment + """ + fn = None + envdata = None + if not pkgs_to_build: + pkgs_to_build = [] + + if buildfile: + # Parse the configuration here. We need to do it explicitly here since + # this showEnvironment() code path doesn't use the cache + self.parseConfiguration() + + fn, cls = bb.cache.Cache.virtualfn2realfn(buildfile) + fn = self.matchFile(fn) + fn = bb.cache.Cache.realfn2virtual(fn, cls) + elif len(pkgs_to_build) == 1: + ignore = self.expanded_data.getVar("ASSUME_PROVIDED", True) or "" + if pkgs_to_build[0] in set(ignore.split()): + bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) + + taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True) + + targetid = taskdata.getbuild_id(pkgs_to_build[0]) + fnid = taskdata.build_targets[targetid][0] + fn = taskdata.fn_index[fnid] + else: + envdata = self.data + + if fn: + try: + envdata = bb.cache.Cache.loadDataFull(fn, self.collection.get_file_appends(fn), self.data) + except Exception as e: + parselog.exception("Unable to read %s", fn) + raise + + # Display history + with closing(StringIO()) as env: + self.data.inchistory.emit(env) + logger.plain(env.getvalue()) + + # emit variables and shell functions + data.update_data(envdata) + with closing(StringIO()) as env: + data.emit_env(env, envdata, True) + logger.plain(env.getvalue()) + + # emit the metadata which isnt valid shell + data.expandKeys(envdata) + for e in envdata.keys(): + if data.getVarFlag( e, 'python', envdata ): + logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False)) + + + def buildTaskData(self, pkgs_to_build, task, abort, allowincomplete=False): + """ + Prepare a runqueue and taskdata object for iteration over pkgs_to_build + """ + bb.event.fire(bb.event.TreeDataPreparationStarted(), self.data) + + # A task of None means use the default task + if task is None: + task = self.configuration.cmd + + fulltargetlist = self.checkPackages(pkgs_to_build) + + localdata = data.createCopy(self.data) + bb.data.update_data(localdata) + bb.data.expandKeys(localdata) + taskdata = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete) + + current = 0 + runlist = [] + for k in fulltargetlist: + ktask = task + if ":do_" in k: + k2 = k.split(":do_") + k = k2[0] + ktask = k2[1] + taskdata.add_provider(localdata, self.recipecache, k) + current += 1 + if not ktask.startswith("do_"): + ktask = "do_%s" % ktask + runlist.append([k, ktask]) + bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data) + taskdata.add_unresolved(localdata, self.recipecache) + bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data) + return taskdata, runlist, fulltargetlist + + def prepareTreeData(self, pkgs_to_build, task): + """ + Prepare a runqueue and taskdata object for iteration over pkgs_to_build + """ + + # We set abort to False here to prevent unbuildable targets raising + # an exception when we're just generating data + taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True) + + return runlist, taskdata + + ######## WARNING : this function requires cache_extra to be enabled ######## + + def generateTaskDepTreeData(self, pkgs_to_build, task): + """ + Create a dependency graph of pkgs_to_build including reverse dependency + information. + """ + runlist, taskdata = self.prepareTreeData(pkgs_to_build, task) + rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist) + rq.rqdata.prepare() + return self.buildDependTree(rq, taskdata) + + + def buildDependTree(self, rq, taskdata): + seen_fnids = [] + depend_tree = {} + depend_tree["depends"] = {} + depend_tree["tdepends"] = {} + depend_tree["pn"] = {} + depend_tree["rdepends-pn"] = {} + depend_tree["packages"] = {} + depend_tree["rdepends-pkg"] = {} + depend_tree["rrecs-pkg"] = {} + depend_tree['providermap'] = {} + depend_tree["layer-priorities"] = self.recipecache.bbfile_config_priorities + + for name, fn in taskdata.get_providermap().iteritems(): + pn = self.recipecache.pkg_fn[fn] + if name != pn: + version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] + depend_tree['providermap'][name] = (pn, version) + + for task in xrange(len(rq.rqdata.runq_fnid)): + taskname = rq.rqdata.runq_task[task] + fnid = rq.rqdata.runq_fnid[task] + fn = taskdata.fn_index[fnid] + pn = self.recipecache.pkg_fn[fn] + version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] + if pn not in depend_tree["pn"]: + depend_tree["pn"][pn] = {} + depend_tree["pn"][pn]["filename"] = fn + depend_tree["pn"][pn]["version"] = version + depend_tree["pn"][pn]["inherits"] = self.recipecache.inherits.get(fn, None) + + # if we have extra caches, list all attributes they bring in + extra_info = [] + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'): + cachefields = getattr(cache_class, 'cachefields', []) + extra_info = extra_info + cachefields + + # for all attributes stored, add them to the dependency tree + for ei in extra_info: + depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn] + + + for dep in rq.rqdata.runq_depends[task]: + depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]] + deppn = self.recipecache.pkg_fn[depfn] + dotname = "%s.%s" % (pn, rq.rqdata.runq_task[task]) + if not dotname in depend_tree["tdepends"]: + depend_tree["tdepends"][dotname] = [] + depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.rqdata.runq_task[dep])) + if fnid not in seen_fnids: + seen_fnids.append(fnid) + packages = [] + + depend_tree["depends"][pn] = [] + for dep in taskdata.depids[fnid]: + depend_tree["depends"][pn].append(taskdata.build_names_index[dep]) + + depend_tree["rdepends-pn"][pn] = [] + for rdep in taskdata.rdepids[fnid]: + depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep]) + + rdepends = self.recipecache.rundeps[fn] + for package in rdepends: + depend_tree["rdepends-pkg"][package] = [] + for rdepend in rdepends[package]: + depend_tree["rdepends-pkg"][package].append(rdepend) + packages.append(package) + + rrecs = self.recipecache.runrecs[fn] + for package in rrecs: + depend_tree["rrecs-pkg"][package] = [] + for rdepend in rrecs[package]: + depend_tree["rrecs-pkg"][package].append(rdepend) + if not package in packages: + packages.append(package) + + for package in packages: + if package not in depend_tree["packages"]: + depend_tree["packages"][package] = {} + depend_tree["packages"][package]["pn"] = pn + depend_tree["packages"][package]["filename"] = fn + depend_tree["packages"][package]["version"] = version + + return depend_tree + + ######## WARNING : this function requires cache_extra to be enabled ######## + def generatePkgDepTreeData(self, pkgs_to_build, task): + """ + Create a dependency tree of pkgs_to_build, returning the data. + """ + _, taskdata = self.prepareTreeData(pkgs_to_build, task) + tasks_fnid = [] + if len(taskdata.tasks_name) != 0: + for task in xrange(len(taskdata.tasks_name)): + tasks_fnid.append(taskdata.tasks_fnid[task]) + + seen_fnids = [] + depend_tree = {} + depend_tree["depends"] = {} + depend_tree["pn"] = {} + depend_tree["rdepends-pn"] = {} + depend_tree["rdepends-pkg"] = {} + depend_tree["rrecs-pkg"] = {} + + # if we have extra caches, list all attributes they bring in + extra_info = [] + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'): + cachefields = getattr(cache_class, 'cachefields', []) + extra_info = extra_info + cachefields + + for task in xrange(len(tasks_fnid)): + fnid = tasks_fnid[task] + fn = taskdata.fn_index[fnid] + pn = self.recipecache.pkg_fn[fn] + + if pn not in depend_tree["pn"]: + depend_tree["pn"][pn] = {} + depend_tree["pn"][pn]["filename"] = fn + version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] + depend_tree["pn"][pn]["version"] = version + rdepends = self.recipecache.rundeps[fn] + rrecs = self.recipecache.runrecs[fn] + depend_tree["pn"][pn]["inherits"] = self.recipecache.inherits.get(fn, None) + + # for all extra attributes stored, add them to the dependency tree + for ei in extra_info: + depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn] + + if fnid not in seen_fnids: + seen_fnids.append(fnid) + + depend_tree["depends"][pn] = [] + for dep in taskdata.depids[fnid]: + item = taskdata.build_names_index[dep] + pn_provider = "" + targetid = taskdata.getbuild_id(item) + if targetid in taskdata.build_targets and taskdata.build_targets[targetid]: + id = taskdata.build_targets[targetid][0] + fn_provider = taskdata.fn_index[id] + pn_provider = self.recipecache.pkg_fn[fn_provider] + else: + pn_provider = item + depend_tree["depends"][pn].append(pn_provider) + + depend_tree["rdepends-pn"][pn] = [] + for rdep in taskdata.rdepids[fnid]: + item = taskdata.run_names_index[rdep] + pn_rprovider = "" + targetid = taskdata.getrun_id(item) + if targetid in taskdata.run_targets and taskdata.run_targets[targetid]: + id = taskdata.run_targets[targetid][0] + fn_rprovider = taskdata.fn_index[id] + pn_rprovider = self.recipecache.pkg_fn[fn_rprovider] + else: + pn_rprovider = item + depend_tree["rdepends-pn"][pn].append(pn_rprovider) + + depend_tree["rdepends-pkg"].update(rdepends) + depend_tree["rrecs-pkg"].update(rrecs) + + return depend_tree + + def generateDepTreeEvent(self, pkgs_to_build, task): + """ + Create a task dependency graph of pkgs_to_build. + Generate an event with the result + """ + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task) + bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.data) + + def generateDotGraphFiles(self, pkgs_to_build, task): + """ + Create a task dependency graph of pkgs_to_build. + Save the result to a set of .dot files. + """ + + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task) + + # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn + depends_file = file('pn-depends.dot', 'w' ) + buildlist_file = file('pn-buildlist', 'w' ) + print("digraph depends {", file=depends_file) + for pn in depgraph["pn"]: + fn = depgraph["pn"][pn]["filename"] + version = depgraph["pn"][pn]["version"] + print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file) + print("%s" % pn, file=buildlist_file) + buildlist_file.close() + logger.info("PN build list saved to 'pn-buildlist'") + for pn in depgraph["depends"]: + for depend in depgraph["depends"][pn]: + print('"%s" -> "%s" [style=solid]' % (pn, depend), file=depends_file) + for pn in depgraph["rdepends-pn"]: + for rdepend in depgraph["rdepends-pn"][pn]: + print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file) + print("}", file=depends_file) + logger.info("PN dependencies saved to 'pn-depends.dot'") + + depends_file = file('package-depends.dot', 'w' ) + print("digraph depends {", file=depends_file) + for package in depgraph["packages"]: + pn = depgraph["packages"][package]["pn"] + fn = depgraph["packages"][package]["filename"] + version = depgraph["packages"][package]["version"] + if package == pn: + print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file) + else: + print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file) + for depend in depgraph["depends"][pn]: + print('"%s" -> "%s" [style=solid]' % (package, depend), file=depends_file) + for package in depgraph["rdepends-pkg"]: + for rdepend in depgraph["rdepends-pkg"][package]: + print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file) + for package in depgraph["rrecs-pkg"]: + for rdepend in depgraph["rrecs-pkg"][package]: + print('"%s" -> "%s" [style=dotted]' % (package, rdepend), file=depends_file) + print("}", file=depends_file) + logger.info("Package dependencies saved to 'package-depends.dot'") + + tdepends_file = file('task-depends.dot', 'w' ) + print("digraph depends {", file=tdepends_file) + for task in depgraph["tdepends"]: + (pn, taskname) = task.rsplit(".", 1) + fn = depgraph["pn"][pn]["filename"] + version = depgraph["pn"][pn]["version"] + print('"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn), file=tdepends_file) + for dep in depgraph["tdepends"][task]: + print('"%s" -> "%s"' % (task, dep), file=tdepends_file) + print("}", file=tdepends_file) + logger.info("Task dependencies saved to 'task-depends.dot'") + + def show_appends_with_no_recipes(self): + # Determine which bbappends haven't been applied + + # First get list of recipes, including skipped + recipefns = self.recipecache.pkg_fn.keys() + recipefns.extend(self.skiplist.keys()) + + # Work out list of bbappends that have been applied + applied_appends = [] + for fn in recipefns: + applied_appends.extend(self.collection.get_file_appends(fn)) + + appends_without_recipes = [] + for _, appendfn in self.collection.bbappends: + if not appendfn in applied_appends: + appends_without_recipes.append(appendfn) + + if appends_without_recipes: + msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes) + warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \ + False) or "no" + if warn_only.lower() in ("1", "yes", "true"): + bb.warn(msg) + else: + bb.fatal(msg) + + def handlePrefProviders(self): + + localdata = data.createCopy(self.data) + bb.data.update_data(localdata) + bb.data.expandKeys(localdata) + + # Handle PREFERRED_PROVIDERS + for p in (localdata.getVar('PREFERRED_PROVIDERS', True) or "").split(): + try: + (providee, provider) = p.split(':') + except: + providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p) + continue + if providee in self.recipecache.preferred and self.recipecache.preferred[providee] != provider: + providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.recipecache.preferred[providee]) + self.recipecache.preferred[providee] = provider + + def findCoreBaseFiles(self, subdir, configfile): + corebase = self.data.getVar('COREBASE', True) or "" + paths = [] + for root, dirs, files in os.walk(corebase + '/' + subdir): + for d in dirs: + configfilepath = os.path.join(root, d, configfile) + if os.path.exists(configfilepath): + paths.append(os.path.join(root, d)) + + if paths: + bb.event.fire(bb.event.CoreBaseFilesFound(paths), self.data) + + def findConfigFilePath(self, configfile): + """ + Find the location on disk of configfile and if it exists and was parsed by BitBake + emit the ConfigFilePathFound event with the path to the file. + """ + path = bb.cookerdata.findConfigFile(configfile, self.data) + if not path: + return + + # Generate a list of parsed configuration files by searching the files + # listed in the __depends and __base_depends variables with a .conf suffix. + conffiles = [] + dep_files = self.data.getVar('__base_depends', False) or [] + dep_files = dep_files + (self.data.getVar('__depends', False) or []) + + for f in dep_files: + if f[0].endswith(".conf"): + conffiles.append(f[0]) + + _, conf, conffile = path.rpartition("conf/") + match = os.path.join(conf, conffile) + # Try and find matches for conf/conffilename.conf as we don't always + # have the full path to the file. + for cfg in conffiles: + if cfg.endswith(match): + bb.event.fire(bb.event.ConfigFilePathFound(path), + self.data) + break + + def findFilesMatchingInDir(self, filepattern, directory): + """ + Searches for files containing the substring 'filepattern' which are children of + 'directory' in each BBPATH. i.e. to find all rootfs package classes available + to BitBake one could call findFilesMatchingInDir(self, 'rootfs_', 'classes') + or to find all machine configuration files one could call: + findFilesMatchingInDir(self, '.conf', 'conf/machine') + """ + + matches = [] + bbpaths = self.data.getVar('BBPATH', True).split(':') + for path in bbpaths: + dirpath = os.path.join(path, directory) + if os.path.exists(dirpath): + for root, dirs, files in os.walk(dirpath): + for f in files: + if filepattern in f: + matches.append(f) + + if matches: + bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data) + + def findConfigFiles(self, varname): + """ + Find config files which are appropriate values for varname. + i.e. MACHINE, DISTRO + """ + possible = [] + var = varname.lower() + + data = self.data + # iterate configs + bbpaths = data.getVar('BBPATH', True).split(':') + for path in bbpaths: + confpath = os.path.join(path, "conf", var) + if os.path.exists(confpath): + for root, dirs, files in os.walk(confpath): + # get all child files, these are appropriate values + for f in files: + val, sep, end = f.rpartition('.') + if end == 'conf': + possible.append(val) + + if possible: + bb.event.fire(bb.event.ConfigFilesFound(var, possible), self.data) + + def findInheritsClass(self, klass): + """ + Find all recipes which inherit the specified class + """ + pkg_list = [] + + for pfn in self.recipecache.pkg_fn: + inherits = self.recipecache.inherits.get(pfn, None) + if inherits and klass in inherits: + pkg_list.append(self.recipecache.pkg_fn[pfn]) + + return pkg_list + + def generateTargetsTree(self, klass=None, pkgs=None): + """ + Generate a dependency tree of buildable targets + Generate an event with the result + """ + # if the caller hasn't specified a pkgs list default to universe + if not pkgs: + pkgs = ['universe'] + # if inherited_class passed ensure all recipes which inherit the + # specified class are included in pkgs + if klass: + extra_pkgs = self.findInheritsClass(klass) + pkgs = pkgs + extra_pkgs + + # generate a dependency tree for all our packages + tree = self.generatePkgDepTreeData(pkgs, 'build') + bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.data) + + def interactiveMode( self ): + """Drop off into a shell""" + try: + from bb import shell + except ImportError: + parselog.exception("Interactive mode not available") + sys.exit(1) + else: + shell.start( self ) + + + def handleCollections( self, collections ): + """Handle collections""" + errors = False + self.recipecache.bbfile_config_priorities = [] + if collections: + collection_priorities = {} + collection_depends = {} + collection_list = collections.split() + min_prio = 0 + for c in collection_list: + # Get collection priority if defined explicitly + priority = self.data.getVar("BBFILE_PRIORITY_%s" % c, True) + if priority: + try: + prio = int(priority) + except ValueError: + parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority) + errors = True + if min_prio == 0 or prio < min_prio: + min_prio = prio + collection_priorities[c] = prio + else: + collection_priorities[c] = None + + # Check dependencies and store information for priority calculation + deps = self.data.getVar("LAYERDEPENDS_%s" % c, True) + if deps: + try: + deplist = bb.utils.explode_dep_versions2(deps) + except bb.utils.VersionStringException as vse: + bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse))) + for dep, oplist in deplist.iteritems(): + if dep in collection_list: + for opstr in oplist: + layerver = self.data.getVar("LAYERVERSION_%s" % dep, True) + (op, depver) = opstr.split() + if layerver: + try: + res = bb.utils.vercmp_string_op(layerver, depver, op) + except bb.utils.VersionStringException as vse: + bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse))) + if not res: + parselog.error("Layer '%s' depends on version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep, layerver) + errors = True + else: + parselog.error("Layer '%s' depends on version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep) + errors = True + else: + parselog.error("Layer '%s' depends on layer '%s', but this layer is not enabled in your configuration", c, dep) + errors = True + collection_depends[c] = deplist.keys() + else: + collection_depends[c] = [] + + # Recursively work out collection priorities based on dependencies + def calc_layer_priority(collection): + if not collection_priorities[collection]: + max_depprio = min_prio + for dep in collection_depends[collection]: + calc_layer_priority(dep) + depprio = collection_priorities[dep] + if depprio > max_depprio: + max_depprio = depprio + max_depprio += 1 + parselog.debug(1, "Calculated priority of layer %s as %d", collection, max_depprio) + collection_priorities[collection] = max_depprio + + # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities + for c in collection_list: + calc_layer_priority(c) + regex = self.data.getVar("BBFILE_PATTERN_%s" % c, True) + if regex == None: + parselog.error("BBFILE_PATTERN_%s not defined" % c) + errors = True + continue + try: + cre = re.compile(regex) + except re.error: + parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex) + errors = True + continue + self.recipecache.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c])) + if errors: + # We've already printed the actual error(s) + raise CollectionError("Errors during parsing layer configuration") + + def buildSetVars(self): + """ + Setup any variables needed before starting a build + """ + t = time.gmtime() + if not self.data.getVar("BUILDNAME", False): + self.data.setVar("BUILDNAME", "${DATE}${TIME}") + self.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', t)) + self.data.setVar("DATE", time.strftime('%Y%m%d', t)) + self.data.setVar("TIME", time.strftime('%H%M%S', t)) + + def matchFiles(self, bf): + """ + Find the .bb files which match the expression in 'buildfile'. + """ + if bf.startswith("/") or bf.startswith("../"): + bf = os.path.abspath(bf) + + self.collection = CookerCollectFiles(self.recipecache.bbfile_config_priorities) + filelist, masked = self.collection.collect_bbfiles(self.data, self.expanded_data) + try: + os.stat(bf) + bf = os.path.abspath(bf) + return [bf] + except OSError: + regexp = re.compile(bf) + matches = [] + for f in filelist: + if regexp.search(f) and os.path.isfile(f): + matches.append(f) + return matches + + def matchFile(self, buildfile): + """ + Find the .bb file which matches the expression in 'buildfile'. + Raise an error if multiple files + """ + matches = self.matchFiles(buildfile) + if len(matches) != 1: + if matches: + msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) + if matches: + for f in matches: + msg += "\n %s" % f + parselog.error(msg) + else: + parselog.error("Unable to find any recipe file matching '%s'" % buildfile) + raise NoSpecificMatch + return matches[0] + + def buildFile(self, buildfile, task): + """ + Build the file matching regexp buildfile + """ + + # Too many people use -b because they think it's how you normally + # specify a target to be built, so show a warning + bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.") + + # Parse the configuration here. We need to do it explicitly here since + # buildFile() doesn't use the cache + self.parseConfiguration() + + # If we are told to do the None task then query the default task + if (task == None): + task = self.configuration.cmd + + fn, cls = bb.cache.Cache.virtualfn2realfn(buildfile) + fn = self.matchFile(fn) + + self.buildSetVars() + + infos = bb.cache.Cache.parse(fn, self.collection.get_file_appends(fn), \ + self.data, + self.caches_array) + infos = dict(infos) + + fn = bb.cache.Cache.realfn2virtual(fn, cls) + try: + info_array = infos[fn] + except KeyError: + bb.fatal("%s does not exist" % fn) + + if info_array[0].skipped: + bb.fatal("%s was skipped: %s" % (fn, info_array[0].skipreason)) + + self.recipecache.add_from_recipeinfo(fn, info_array) + + # Tweak some variables + item = info_array[0].pn + self.recipecache.ignored_dependencies = set() + self.recipecache.bbfile_priority[fn] = 1 + + # Remove external dependencies + self.recipecache.task_deps[fn]['depends'] = {} + self.recipecache.deps[fn] = [] + self.recipecache.rundeps[fn] = [] + self.recipecache.runrecs[fn] = [] + + # Invalidate task for target if force mode active + if self.configuration.force: + logger.verbose("Invalidate task %s, %s", task, fn) + if not task.startswith("do_"): + task = "do_%s" % task + bb.parse.siggen.invalidate_task(task, self.recipecache, fn) + + # Setup taskdata structure + taskdata = bb.taskdata.TaskData(self.configuration.abort) + taskdata.add_provider(self.data, self.recipecache, item) + + buildname = self.data.getVar("BUILDNAME", True) + bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.expanded_data) + + # Execute the runqueue + if not task.startswith("do_"): + task = "do_%s" % task + runlist = [[item, task]] + + rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist) + + def buildFileIdle(server, rq, abort): + + msg = None + interrupted = 0 + if abort or self.state == state.forceshutdown: + rq.finish_runqueue(True) + msg = "Forced shutdown" + interrupted = 2 + elif self.state == state.shutdown: + rq.finish_runqueue(False) + msg = "Stopped build" + interrupted = 1 + failures = 0 + try: + retval = rq.execute_runqueue() + except runqueue.TaskFailure as exc: + failures += len(exc.args) + retval = False + except SystemExit as exc: + self.command.finishAsyncCommand(str(exc)) + return False + + if not retval: + bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures, interrupted), self.expanded_data) + self.command.finishAsyncCommand(msg) + return False + if retval is True: + return True + return retval + + self.configuration.server_register_idlecallback(buildFileIdle, rq) + + def buildTargets(self, targets, task): + """ + Attempt to build the targets specified + """ + + def buildTargetsIdle(server, rq, abort): + msg = None + interrupted = 0 + if abort or self.state == state.forceshutdown: + rq.finish_runqueue(True) + msg = "Forced shutdown" + interrupted = 2 + elif self.state == state.shutdown: + rq.finish_runqueue(False) + msg = "Stopped build" + interrupted = 1 + failures = 0 + try: + retval = rq.execute_runqueue() + except runqueue.TaskFailure as exc: + failures += len(exc.args) + retval = False + except SystemExit as exc: + self.command.finishAsyncCommand(str(exc)) + return False + + if not retval: + bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets, failures, interrupted), self.data) + self.command.finishAsyncCommand(msg) + return False + if retval is True: + return True + return retval + + build.reset_cache() + self.buildSetVars() + + # If we are told to do the None task then query the default task + if (task == None): + task = self.configuration.cmd + + if not task.startswith("do_"): + task = "do_%s" % task + + taskdata, runlist, fulltargetlist = self.buildTaskData(targets, task, self.configuration.abort) + + buildname = self.data.getVar("BUILDNAME", False) + + # make targets to always look as <target>:do_<task> + ntargets = [] + for target in fulltargetlist: + if ":" in target: + if ":do_" not in target: + target = "%s:do_%s" % tuple(target.split(":", 1)) + else: + target = "%s:%s" % (target, task) + ntargets.append(target) + + bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.data) + + rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist) + if 'universe' in targets: + rq.rqdata.warn_multi_bb = True + + self.configuration.server_register_idlecallback(buildTargetsIdle, rq) + + + def getAllKeysWithFlags(self, flaglist): + dump = {} + for k in self.data.keys(): + try: + expand = True + flags = self.data.getVarFlags(k) + if flags and "func" in flags and "python" in flags: + expand = False + v = self.data.getVar(k, expand) + if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart): + dump[k] = { + 'v' : v , + 'history' : self.data.varhistory.variable(k), + } + for d in flaglist: + if flags and d in flags: + dump[k][d] = flags[d] + else: + dump[k][d] = None + except Exception as e: + print(e) + return dump + + + def generateNewImage(self, image, base_image, package_queue, timestamp, description): + ''' + Create a new image with a "require"/"inherit" base_image statement + ''' + if timestamp: + image_name = os.path.splitext(image)[0] + timestr = time.strftime("-%Y%m%d-%H%M%S") + dest = image_name + str(timestr) + ".bb" + else: + if not image.endswith(".bb"): + dest = image + ".bb" + else: + dest = image + + basename = False + if base_image: + with open(base_image, 'r') as f: + require_line = f.readline() + p = re.compile("IMAGE_BASENAME *=") + for line in f: + if p.search(line): + basename = True + + with open(dest, "w") as imagefile: + if base_image is None: + imagefile.write("inherit core-image\n") + else: + topdir = self.data.getVar("TOPDIR", False) + if topdir in base_image: + base_image = require_line.split()[1] + imagefile.write("require " + base_image + "\n") + image_install = "IMAGE_INSTALL = \"" + for package in package_queue: + image_install += str(package) + " " + image_install += "\"\n" + imagefile.write(image_install) + + description_var = "DESCRIPTION = \"" + description + "\"\n" + imagefile.write(description_var) + + if basename: + # If this is overwritten in a inherited image, reset it to default + image_basename = "IMAGE_BASENAME = \"${PN}\"\n" + imagefile.write(image_basename) + + self.state = state.initial + if timestamp: + return timestr + + def updateCacheSync(self): + if self.state == state.running: + return + + # reload files for which we got notifications + for p in self.inotify_modified_files: + bb.parse.update_cache(p) + if p in bb.parse.BBHandler.cached_statements: + del bb.parse.BBHandler.cached_statements[p] + self.inotify_modified_files = [] + + if not self.baseconfig_valid: + logger.debug(1, "Reloading base configuration data") + self.initConfigurationData() + self.baseconfig_valid = True + self.parsecache_valid = False + + # This is called for all async commands when self.state != running + def updateCache(self): + if self.state == state.running: + return + + if self.state in (state.shutdown, state.forceshutdown, state.error): + if hasattr(self.parser, 'shutdown'): + self.parser.shutdown(clean=False, force = True) + raise bb.BBHandledException() + + if self.state != state.parsing: + self.updateCacheSync() + + if self.state != state.parsing and not self.parsecache_valid: + self.parseConfiguration () + if CookerFeatures.SEND_SANITYEVENTS in self.featureset: + bb.event.fire(bb.event.SanityCheck(False), self.data) + + ignore = self.expanded_data.getVar("ASSUME_PROVIDED", True) or "" + self.recipecache.ignored_dependencies = set(ignore.split()) + + for dep in self.configuration.extra_assume_provided: + self.recipecache.ignored_dependencies.add(dep) + + self.collection = CookerCollectFiles(self.recipecache.bbfile_config_priorities) + (filelist, masked) = self.collection.collect_bbfiles(self.data, self.expanded_data) + + self.parser = CookerParser(self, filelist, masked) + self.parsecache_valid = True + + self.state = state.parsing + + if not self.parser.parse_next(): + collectlog.debug(1, "parsing complete") + if self.parser.error: + raise bb.BBHandledException() + self.show_appends_with_no_recipes() + self.handlePrefProviders() + self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn, self.data) + self.state = state.running + + # Send an event listing all stamps reachable after parsing + # which the metadata may use to clean up stale data + event = bb.event.ReachableStamps(self.recipecache.stamp) + bb.event.fire(event, self.expanded_data) + return None + + return True + + def checkPackages(self, pkgs_to_build): + + # Return a copy, don't modify the original + pkgs_to_build = pkgs_to_build[:] + + if len(pkgs_to_build) == 0: + raise NothingToBuild + + ignore = (self.expanded_data.getVar("ASSUME_PROVIDED", True) or "").split() + for pkg in pkgs_to_build: + if pkg in ignore: + parselog.warn("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg) + + if 'world' in pkgs_to_build: + bb.providers.buildWorldTargetList(self.recipecache) + pkgs_to_build.remove('world') + for t in self.recipecache.world_target: + pkgs_to_build.append(t) + + if 'universe' in pkgs_to_build: + parselog.warn("The \"universe\" target is only intended for testing and may produce errors.") + parselog.debug(1, "collating packages for \"universe\"") + pkgs_to_build.remove('universe') + for t in self.recipecache.universe_target: + pkgs_to_build.append(t) + + return pkgs_to_build + + + + + def pre_serve(self): + # Empty the environment. The environment will be populated as + # necessary from the data store. + #bb.utils.empty_environment() + try: + self.prhost = prserv.serv.auto_start(self.data) + except prserv.serv.PRServiceConfigError: + bb.event.fire(CookerExit(), self.expanded_data) + self.state = state.error + return + + def post_serve(self): + prserv.serv.auto_shutdown(self.data) + bb.event.fire(CookerExit(), self.expanded_data) + lockfile = self.lock.name + self.lock.close() + self.lock = None + + while not self.lock: + with bb.utils.timeout(3): + self.lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True) + if not self.lock: + # Some systems may not have lsof available + procs = None + try: + procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT) + except OSError as e: + if e.errno != errno.ENOENT: + raise + if procs is None: + # Fall back to fuser if lsof is unavailable + try: + procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock" + if procs: + msg += ":\n%s" % str(procs) + print(msg) + + + def shutdown(self, force = False): + if force: + self.state = state.forceshutdown + else: + self.state = state.shutdown + + if self.parser: + self.parser.shutdown(clean=not force, force=force) + + def finishcommand(self): + self.state = state.initial + + def reset(self): + self.initConfigurationData() + + def lockBitbake(self): + if not hasattr(self, 'lock'): + self.lock = None + if self.data: + lockfile = self.data.expand("${TOPDIR}/bitbake.lock") + if lockfile: + self.lock = bb.utils.lockfile(lockfile, False, False) + return self.lock + + def unlockBitbake(self): + if hasattr(self, 'lock') and self.lock: + bb.utils.unlockfile(self.lock) + +def server_main(cooker, func, *args): + cooker.pre_serve() + + if cooker.configuration.profile: + try: + import cProfile as profile + except: + import profile + prof = profile.Profile() + + ret = profile.Profile.runcall(prof, func, *args) + + prof.dump_stats("profile.log") + bb.utils.process_profilelog("profile.log") + print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed") + + else: + ret = func(*args) + + cooker.post_serve() + + return ret + +class CookerExit(bb.event.Event): + """ + Notify clients of the Cooker shutdown + """ + + def __init__(self): + bb.event.Event.__init__(self) + + +class CookerCollectFiles(object): + def __init__(self, priorities): + self.bbappends = [] + self.bbfile_config_priorities = priorities + + def calc_bbfile_priority( self, filename, matched = None ): + for _, _, regex, pri in self.bbfile_config_priorities: + if regex.match(filename): + if matched != None: + if not regex in matched: + matched.add(regex) + return pri + return 0 + + def get_bbfiles(self): + """Get list of default .bb files by reading out the current directory""" + path = os.getcwd() + contents = os.listdir(path) + bbfiles = [] + for f in contents: + if f.endswith(".bb"): + bbfiles.append(os.path.abspath(os.path.join(path, f))) + return bbfiles + + def find_bbfiles(self, path): + """Find all the .bb and .bbappend files in a directory""" + found = [] + for dir, dirs, files in os.walk(path): + for ignored in ('SCCS', 'CVS', '.svn'): + if ignored in dirs: + dirs.remove(ignored) + found += [os.path.join(dir, f) for f in files if (f.endswith(['.bb', '.bbappend']))] + + return found + + def collect_bbfiles(self, config, eventdata): + """Collect all available .bb build files""" + masked = 0 + + collectlog.debug(1, "collecting .bb files") + + files = (config.getVar( "BBFILES", True) or "").split() + config.setVar("BBFILES", " ".join(files)) + + # Sort files by priority + files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) ) + + if not len(files): + files = self.get_bbfiles() + + if not len(files): + collectlog.error("no recipe files to build, check your BBPATH and BBFILES?") + bb.event.fire(CookerExit(), eventdata) + + # Can't use set here as order is important + newfiles = [] + for f in files: + if os.path.isdir(f): + dirfiles = self.find_bbfiles(f) + for g in dirfiles: + if g not in newfiles: + newfiles.append(g) + else: + globbed = glob.glob(f) + if not globbed and os.path.exists(f): + globbed = [f] + # glob gives files in order on disk. Sort to be deterministic. + for g in sorted(globbed): + if g not in newfiles: + newfiles.append(g) + + bbmask = config.getVar('BBMASK', True) + + if bbmask: + # First validate the individual regular expressions and ignore any + # that do not compile + bbmasks = [] + for mask in bbmask.split(): + try: + re.compile(mask) + bbmasks.append(mask) + except sre_constants.error: + collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask) + + # Then validate the combined regular expressions. This should never + # fail, but better safe than sorry... + bbmask = "|".join(bbmasks) + try: + bbmask_compiled = re.compile(bbmask) + except sre_constants.error: + collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask) + bbmask = None + + bbfiles = [] + bbappend = [] + for f in newfiles: + if bbmask and bbmask_compiled.search(f): + collectlog.debug(1, "skipping masked file %s", f) + masked += 1 + continue + if f.endswith('.bb'): + bbfiles.append(f) + elif f.endswith('.bbappend'): + bbappend.append(f) + else: + collectlog.debug(1, "skipping %s: unknown file extension", f) + + # Build a list of .bbappend files for each .bb file + for f in bbappend: + base = os.path.basename(f).replace('.bbappend', '.bb') + self.bbappends.append((base, f)) + + # Find overlayed recipes + # bbfiles will be in priority order which makes this easy + bbfile_seen = dict() + self.overlayed = defaultdict(list) + for f in reversed(bbfiles): + base = os.path.basename(f) + if base not in bbfile_seen: + bbfile_seen[base] = f + else: + topfile = bbfile_seen[base] + self.overlayed[topfile].append(f) + + return (bbfiles, masked) + + def get_file_appends(self, fn): + """ + Returns a list of .bbappend files to apply to fn + """ + filelist = [] + f = os.path.basename(fn) + for b in self.bbappends: + (bbappend, filename) = b + if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): + filelist.append(filename) + return filelist + + def collection_priorities(self, pkgfns, d): + + priorities = {} + + # Calculate priorities for each file + matched = set() + for p in pkgfns: + realfn, cls = bb.cache.Cache.virtualfn2realfn(p) + priorities[p] = self.calc_bbfile_priority(realfn, matched) + + # Don't show the warning if the BBFILE_PATTERN did match .bbappend files + unmatched = set() + for _, _, regex, pri in self.bbfile_config_priorities: + if not regex in matched: + unmatched.add(regex) + + def findmatch(regex): + for b in self.bbappends: + (bbfile, append) = b + if regex.match(append): + return True + return False + + for unmatch in unmatched.copy(): + if findmatch(unmatch): + unmatched.remove(unmatch) + + for collection, pattern, regex, _ in self.bbfile_config_priorities: + if regex in unmatched: + if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection, True) != '1': + collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) + + return priorities + +class ParsingFailure(Exception): + def __init__(self, realexception, recipe): + self.realexception = realexception + self.recipe = recipe + Exception.__init__(self, realexception, recipe) + +class Feeder(multiprocessing.Process): + def __init__(self, jobs, to_parsers, quit): + self.quit = quit + self.jobs = jobs + self.to_parsers = to_parsers + multiprocessing.Process.__init__(self) + + def run(self): + while True: + try: + quit = self.quit.get_nowait() + except Queue.Empty: + pass + else: + if quit == 'cancel': + self.to_parsers.cancel_join_thread() + break + + try: + job = self.jobs.pop() + except IndexError: + break + + try: + self.to_parsers.put(job, timeout=0.5) + except Queue.Full: + self.jobs.insert(0, job) + continue + +class Parser(multiprocessing.Process): + def __init__(self, jobs, results, quit, init, profile): + self.jobs = jobs + self.results = results + self.quit = quit + self.init = init + multiprocessing.Process.__init__(self) + self.context = bb.utils.get_context().copy() + self.handlers = bb.event.get_class_handlers().copy() + self.profile = profile + + def run(self): + + if not self.profile: + self.realrun() + return + + try: + import cProfile as profile + except: + import profile + prof = profile.Profile() + try: + profile.Profile.runcall(prof, self.realrun) + finally: + logfile = "profile-parse-%s.log" % multiprocessing.current_process().name + prof.dump_stats(logfile) + + def realrun(self): + if self.init: + self.init() + + pending = [] + while True: + try: + self.quit.get_nowait() + except Queue.Empty: + pass + else: + self.results.cancel_join_thread() + break + + if pending: + result = pending.pop() + else: + try: + job = self.jobs.get(timeout=0.25) + except Queue.Empty: + continue + + if job is None: + break + result = self.parse(*job) + + try: + self.results.put(result, timeout=0.25) + except Queue.Full: + pending.append(result) + + def parse(self, filename, appends, caches_array): + try: + # Record the filename we're parsing into any events generated + def parse_filter(self, record): + record.taskpid = bb.event.worker_pid + record.fn = filename + return True + + # Reset our environment and handlers to the original settings + bb.utils.set_context(self.context.copy()) + bb.event.set_class_handlers(self.handlers.copy()) + bb.event.LogHandler.filter = parse_filter + + return True, bb.cache.Cache.parse(filename, appends, self.cfg, caches_array) + except Exception as exc: + tb = sys.exc_info()[2] + exc.recipe = filename + exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3)) + return True, exc + # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown + # and for example a worker thread doesn't just exit on its own in response to + # a SystemExit event for example. + except BaseException as exc: + return True, ParsingFailure(exc, filename) + +class CookerParser(object): + def __init__(self, cooker, filelist, masked): + self.filelist = filelist + self.cooker = cooker + self.cfgdata = cooker.data + self.cfghash = cooker.data_hash + + # Accounting statistics + self.parsed = 0 + self.cached = 0 + self.error = 0 + self.masked = masked + + self.skipped = 0 + self.virtuals = 0 + self.total = len(filelist) + + self.current = 0 + self.process_names = [] + + self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array) + self.fromcache = [] + self.willparse = [] + for filename in self.filelist: + appends = self.cooker.collection.get_file_appends(filename) + if not self.bb_cache.cacheValid(filename, appends): + self.willparse.append((filename, appends, cooker.caches_array)) + else: + self.fromcache.append((filename, appends)) + self.toparse = self.total - len(self.fromcache) + self.progress_chunk = max(self.toparse / 100, 1) + + self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or + multiprocessing.cpu_count()), len(self.willparse)) + + self.start() + self.haveshutdown = False + + def start(self): + self.results = self.load_cached() + self.processes = [] + if self.toparse: + bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) + def init(): + Parser.cfg = self.cfgdata + bb.utils.set_process_name(multiprocessing.current_process().name) + multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) + multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) + + self.feeder_quit = multiprocessing.Queue(maxsize=1) + self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) + self.jobs = multiprocessing.Queue(maxsize=self.num_processes) + self.result_queue = multiprocessing.Queue() + self.feeder = Feeder(self.willparse, self.jobs, self.feeder_quit) + self.feeder.start() + for i in range(0, self.num_processes): + parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) + parser.start() + self.process_names.append(parser.name) + self.processes.append(parser) + + self.results = itertools.chain(self.results, self.parse_generator()) + + def shutdown(self, clean=True, force=False): + if not self.toparse: + return + if self.haveshutdown: + return + self.haveshutdown = True + + if clean: + event = bb.event.ParseCompleted(self.cached, self.parsed, + self.skipped, self.masked, + self.virtuals, self.error, + self.total) + + bb.event.fire(event, self.cfgdata) + self.feeder_quit.put(None) + for process in self.processes: + self.parser_quit.put(None) + else: + self.feeder_quit.put('cancel') + + self.parser_quit.cancel_join_thread() + for process in self.processes: + self.parser_quit.put(None) + + self.jobs.cancel_join_thread() + + for process in self.processes: + if force: + process.join(.1) + process.terminate() + else: + process.join() + self.feeder.join() + + sync = threading.Thread(target=self.bb_cache.sync) + sync.start() + multiprocessing.util.Finalize(None, sync.join, exitpriority=-100) + bb.codeparser.parser_cache_savemerge() + bb.fetch.fetcher_parse_done() + if self.cooker.configuration.profile: + profiles = [] + for i in self.process_names: + logfile = "profile-parse-%s.log" % i + if os.path.exists(logfile): + profiles.append(logfile) + + pout = "profile-parse.log.processed" + bb.utils.process_profilelog(profiles, pout = pout) + print("Processed parsing statistics saved to %s" % (pout)) + + def load_cached(self): + for filename, appends in self.fromcache: + cached, infos = self.bb_cache.load(filename, appends, self.cfgdata) + yield not cached, infos + + def parse_generator(self): + while True: + if self.parsed >= self.toparse: + break + + try: + result = self.result_queue.get(timeout=0.25) + except Queue.Empty: + pass + else: + value = result[1] + if isinstance(value, BaseException): + raise value + else: + yield result + + def parse_next(self): + result = [] + parsed = None + try: + parsed, result = self.results.next() + except StopIteration: + self.shutdown() + return False + except bb.BBHandledException as exc: + self.error += 1 + logger.error('Failed to parse recipe: %s' % exc.recipe) + self.shutdown(clean=False) + return False + except ParsingFailure as exc: + self.error += 1 + logger.error('Unable to parse %s: %s' % + (exc.recipe, bb.exceptions.to_string(exc.realexception))) + self.shutdown(clean=False) + return False + except bb.parse.ParseError as exc: + self.error += 1 + logger.error(str(exc)) + self.shutdown(clean=False) + return False + except bb.data_smart.ExpansionError as exc: + self.error += 1 + _, value, _ = sys.exc_info() + logger.error('ExpansionError during parsing %s: %s', value.recipe, str(exc)) + self.shutdown(clean=False) + return False + except Exception as exc: + self.error += 1 + etype, value, tb = sys.exc_info() + if hasattr(value, "recipe"): + logger.error('Unable to parse %s', value.recipe, + exc_info=(etype, value, exc.traceback)) + else: + # Most likely, an exception occurred during raising an exception + import traceback + logger.error('Exception during parse: %s' % traceback.format_exc()) + self.shutdown(clean=False) + return False + + self.current += 1 + self.virtuals += len(result) + if parsed: + self.parsed += 1 + if self.parsed % self.progress_chunk == 0: + bb.event.fire(bb.event.ParseProgress(self.parsed, self.toparse), + self.cfgdata) + else: + self.cached += 1 + + for virtualfn, info_array in result: + if info_array[0].skipped: + self.skipped += 1 + self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) + self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecache, + parsed=parsed, watcher = self.cooker.add_filewatch) + return True + + def reparse(self, filename): + infos = self.bb_cache.parse(filename, + self.cooker.collection.get_file_appends(filename), + self.cfgdata, self.cooker.caches_array) + for vfn, info_array in infos: + self.cooker.recipecache.add_from_recipeinfo(vfn, info_array) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py b/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py new file mode 100644 index 000000000..50259a9a0 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH +# Copyright (C) 2006 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os, sys +from functools import wraps +import logging +import bb +from bb import data +import bb.parse + +logger = logging.getLogger("BitBake") +parselog = logging.getLogger("BitBake.Parsing") + +class ConfigParameters(object): + def __init__(self, argv=sys.argv): + self.options, targets = self.parseCommandLine(argv) + self.environment = self.parseEnvironment() + + self.options.pkgs_to_build = targets or [] + + self.options.tracking = False + if hasattr(self.options, "show_environment") and self.options.show_environment: + self.options.tracking = True + + for key, val in self.options.__dict__.items(): + setattr(self, key, val) + + def parseCommandLine(self, argv=sys.argv): + raise Exception("Caller must implement commandline option parsing") + + def parseEnvironment(self): + return os.environ.copy() + + def updateFromServer(self, server): + if not self.options.cmd: + defaulttask, error = server.runCommand(["getVariable", "BB_DEFAULT_TASK"]) + if error: + raise Exception("Unable to get the value of BB_DEFAULT_TASK from the server: %s" % error) + self.options.cmd = defaulttask or "build" + _, error = server.runCommand(["setConfig", "cmd", self.options.cmd]) + if error: + raise Exception("Unable to set configuration option 'cmd' on the server: %s" % error) + + if not self.options.pkgs_to_build: + bbpkgs, error = server.runCommand(["getVariable", "BBTARGETS"]) + if error: + raise Exception("Unable to get the value of BBTARGETS from the server: %s" % error) + if bbpkgs: + self.options.pkgs_to_build.extend(bbpkgs.split()) + + def updateToServer(self, server, environment): + options = {} + for o in ["abort", "tryaltconfigs", "force", "invalidate_stamp", + "verbose", "debug", "dry_run", "dump_signatures", + "debug_domains", "extra_assume_provided", "profile", + "prefile", "postfile"]: + options[o] = getattr(self.options, o) + + ret, error = server.runCommand(["updateConfig", options, environment]) + if error: + raise Exception("Unable to update the server configuration with local parameters: %s" % error) + + def parseActions(self): + # Parse any commandline into actions + action = {'action':None, 'msg':None} + if self.options.show_environment: + if 'world' in self.options.pkgs_to_build: + action['msg'] = "'world' is not a valid target for --environment." + elif 'universe' in self.options.pkgs_to_build: + action['msg'] = "'universe' is not a valid target for --environment." + elif len(self.options.pkgs_to_build) > 1: + action['msg'] = "Only one target can be used with the --environment option." + elif self.options.buildfile and len(self.options.pkgs_to_build) > 0: + action['msg'] = "No target should be used with the --environment and --buildfile options." + elif len(self.options.pkgs_to_build) > 0: + action['action'] = ["showEnvironmentTarget", self.options.pkgs_to_build] + else: + action['action'] = ["showEnvironment", self.options.buildfile] + elif self.options.buildfile is not None: + action['action'] = ["buildFile", self.options.buildfile, self.options.cmd] + elif self.options.revisions_changed: + action['action'] = ["compareRevisions"] + elif self.options.show_versions: + action['action'] = ["showVersions"] + elif self.options.parse_only: + action['action'] = ["parseFiles"] + elif self.options.dot_graph: + if self.options.pkgs_to_build: + action['action'] = ["generateDotGraph", self.options.pkgs_to_build, self.options.cmd] + else: + action['msg'] = "Please specify a package name for dependency graph generation." + else: + if self.options.pkgs_to_build: + action['action'] = ["buildTargets", self.options.pkgs_to_build, self.options.cmd] + else: + #action['msg'] = "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information." + action = None + self.options.initialaction = action + return action + +class CookerConfiguration(object): + """ + Manages build options and configurations for one run + """ + + def __init__(self): + self.debug_domains = [] + self.extra_assume_provided = [] + self.prefile = [] + self.postfile = [] + self.prefile_server = [] + self.postfile_server = [] + self.debug = 0 + self.cmd = None + self.abort = True + self.force = False + self.profile = False + self.nosetscene = False + self.setsceneonly = False + self.invalidate_stamp = False + self.dump_signatures = [] + self.dry_run = False + self.tracking = False + self.interface = [] + self.writeeventlog = False + + self.env = {} + + def setConfigParameters(self, parameters): + for key in self.__dict__.keys(): + if key in parameters.options.__dict__: + setattr(self, key, parameters.options.__dict__[key]) + self.env = parameters.environment.copy() + self.tracking = parameters.tracking + + def setServerRegIdleCallback(self, srcb): + self.server_register_idlecallback = srcb + + def __getstate__(self): + state = {} + for key in self.__dict__.keys(): + if key == "server_register_idlecallback": + state[key] = None + else: + state[key] = getattr(self, key) + return state + + def __setstate__(self,state): + for k in state: + setattr(self, k, state[k]) + + +def catch_parse_error(func): + """Exception handling bits for our parsing""" + @wraps(func) + def wrapped(fn, *args): + try: + return func(fn, *args) + except IOError as exc: + import traceback + parselog.critical(traceback.format_exc()) + parselog.critical("Unable to parse %s: %s" % (fn, exc)) + sys.exit(1) + except bb.data_smart.ExpansionError as exc: + import traceback + + bbdir = os.path.dirname(__file__) + os.sep + exc_class, exc, tb = sys.exc_info() + for tb in iter(lambda: tb.tb_next, None): + # Skip frames in bitbake itself, we only want the metadata + fn, _, _, _ = traceback.extract_tb(tb, 1)[0] + if not fn.startswith(bbdir): + break + parselog.critical("Unable to parse %s", fn, exc_info=(exc_class, exc, tb)) + except bb.parse.ParseError as exc: + parselog.critical(str(exc)) + sys.exit(1) + return wrapped + +@catch_parse_error +def parse_config_file(fn, data, include=True): + return bb.parse.handle(fn, data, include) + +@catch_parse_error +def _inherit(bbclass, data): + bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data) + return data + +def findConfigFile(configfile, data): + search = [] + bbpath = data.getVar("BBPATH", True) + if bbpath: + for i in bbpath.split(":"): + search.append(os.path.join(i, "conf", configfile)) + path = os.getcwd() + while path != "/": + search.append(os.path.join(path, "conf", configfile)) + path, _ = os.path.split(path) + + for i in search: + if os.path.exists(i): + return i + + return None + +class CookerDataBuilder(object): + + def __init__(self, cookercfg, worker = False): + + self.prefiles = cookercfg.prefile + self.postfiles = cookercfg.postfile + self.tracking = cookercfg.tracking + + bb.utils.set_context(bb.utils.clean_context()) + bb.event.set_class_handlers(bb.event.clean_class_handlers()) + self.data = bb.data.init() + if self.tracking: + self.data.enableTracking() + + # Keep a datastore of the initial environment variables and their + # values from when BitBake was launched to enable child processes + # to use environment variables which have been cleaned from the + # BitBake processes env + self.savedenv = bb.data.init() + for k in cookercfg.env: + self.savedenv.setVar(k, cookercfg.env[k]) + + filtered_keys = bb.utils.approved_variables() + bb.data.inheritFromOS(self.data, self.savedenv, filtered_keys) + self.data.setVar("BB_ORIGENV", self.savedenv) + + if worker: + self.data.setVar("BB_WORKERCONTEXT", "1") + + def parseBaseConfiguration(self): + try: + self.parseConfigurationFiles(self.prefiles, self.postfiles) + except SyntaxError: + raise bb.BBHandledException + except bb.data_smart.ExpansionError as e: + logger.error(str(e)) + raise bb.BBHandledException + except Exception: + logger.exception("Error parsing configuration files") + raise bb.BBHandledException + + def _findLayerConf(self, data): + return findConfigFile("bblayers.conf", data) + + def parseConfigurationFiles(self, prefiles, postfiles): + data = self.data + bb.parse.init_parser(data) + + # Parse files for loading *before* bitbake.conf and any includes + for f in prefiles: + data = parse_config_file(f, data) + + layerconf = self._findLayerConf(data) + if layerconf: + parselog.debug(2, "Found bblayers.conf (%s)", layerconf) + # By definition bblayers.conf is in conf/ of TOPDIR. + # We may have been called with cwd somewhere else so reset TOPDIR + data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf))) + data = parse_config_file(layerconf, data) + + layers = (data.getVar('BBLAYERS', True) or "").split() + + data = bb.data.createCopy(data) + approved = bb.utils.approved_variables() + for layer in layers: + parselog.debug(2, "Adding layer %s", layer) + if 'HOME' in approved and '~' in layer: + layer = os.path.expanduser(layer) + if layer.endswith('/'): + layer = layer.rstrip('/') + data.setVar('LAYERDIR', layer) + data = parse_config_file(os.path.join(layer, "conf", "layer.conf"), data) + data.expandVarref('LAYERDIR') + + data.delVar('LAYERDIR') + + if not data.getVar("BBPATH", True): + msg = "The BBPATH variable is not set" + if not layerconf: + msg += (" and bitbake did not find a conf/bblayers.conf file in" + " the expected location.\nMaybe you accidentally" + " invoked bitbake from the wrong directory?") + raise SystemExit(msg) + + data = parse_config_file(os.path.join("conf", "bitbake.conf"), data) + + # Parse files for loading *after* bitbake.conf and any includes + for p in postfiles: + data = parse_config_file(p, data) + + # Handle any INHERITs and inherit the base class + bbclasses = ["base"] + (data.getVar('INHERIT', True) or "").split() + for bbclass in bbclasses: + data = _inherit(bbclass, data) + + # Nomally we only register event handlers at the end of parsing .bb files + # We register any handlers we've found so far here... + for var in data.getVar('__BBHANDLERS', False) or []: + handlerfn = data.getVarFlag(var, "filename", False) + handlerln = int(data.getVarFlag(var, "lineno", False)) + bb.event.register(var, data.getVar(var, False), (data.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln) + + if data.getVar("BB_WORKERCONTEXT", False) is None: + bb.fetch.fetcher_init(data) + bb.codeparser.parser_cache_init(data) + bb.event.fire(bb.event.ConfigParsed(), data) + + if data.getVar("BB_INVALIDCONF", False) is True: + data.setVar("BB_INVALIDCONF", False) + self.parseConfigurationFiles(self.prefiles, self.postfiles) + return + + bb.parse.init_parser(data) + data.setVar('BBINCLUDED',bb.parse.get_file_depends(data)) + self.data = data + self.data_hash = data.get_hash() + + + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/daemonize.py b/import-layers/yocto-poky/bitbake/lib/bb/daemonize.py new file mode 100644 index 000000000..346a61858 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/daemonize.py @@ -0,0 +1,193 @@ +""" +Python Daemonizing helper + +Configurable daemon behaviors: + + 1.) The current working directory set to the "/" directory. + 2.) The current file creation mode mask set to 0. + 3.) Close all open files (1024). + 4.) Redirect standard I/O streams to "/dev/null". + +A failed call to fork() now raises an exception. + +References: + 1) Advanced Programming in the Unix Environment: W. Richard Stevens + http://www.apuebook.com/apue3e.html + 2) The Linux Programming Interface: Michael Kerrisk + http://man7.org/tlpi/index.html + 3) Unix Programming Frequently Asked Questions: + http://www.faqs.org/faqs/unix-faq/programmer/faq/ + +Modified to allow a function to be daemonized and return for +bitbake use by Richard Purdie +""" + +__author__ = "Chad J. Schroeder" +__copyright__ = "Copyright (C) 2005 Chad J. Schroeder" +__version__ = "0.2" + +# Standard Python modules. +import os # Miscellaneous OS interfaces. +import sys # System-specific parameters and functions. + +# Default daemon parameters. +# File mode creation mask of the daemon. +# For BitBake's children, we do want to inherit the parent umask. +UMASK = None + +# Default maximum for the number of available file descriptors. +MAXFD = 1024 + +# The standard I/O file descriptors are redirected to /dev/null by default. +if (hasattr(os, "devnull")): + REDIRECT_TO = os.devnull +else: + REDIRECT_TO = "/dev/null" + +def createDaemon(function, logfile): + """ + Detach a process from the controlling terminal and run it in the + background as a daemon, returning control to the caller. + """ + + try: + # Fork a child process so the parent can exit. This returns control to + # the command-line or shell. It also guarantees that the child will not + # be a process group leader, since the child receives a new process ID + # and inherits the parent's process group ID. This step is required + # to insure that the next call to os.setsid is successful. + pid = os.fork() + except OSError as e: + raise Exception("%s [%d]" % (e.strerror, e.errno)) + + if (pid == 0): # The first child. + # To become the session leader of this new session and the process group + # leader of the new process group, we call os.setsid(). The process is + # also guaranteed not to have a controlling terminal. + os.setsid() + + # Is ignoring SIGHUP necessary? + # + # It's often suggested that the SIGHUP signal should be ignored before + # the second fork to avoid premature termination of the process. The + # reason is that when the first child terminates, all processes, e.g. + # the second child, in the orphaned group will be sent a SIGHUP. + # + # "However, as part of the session management system, there are exactly + # two cases where SIGHUP is sent on the death of a process: + # + # 1) When the process that dies is the session leader of a session that + # is attached to a terminal device, SIGHUP is sent to all processes + # in the foreground process group of that terminal device. + # 2) When the death of a process causes a process group to become + # orphaned, and one or more processes in the orphaned group are + # stopped, then SIGHUP and SIGCONT are sent to all members of the + # orphaned group." [2] + # + # The first case can be ignored since the child is guaranteed not to have + # a controlling terminal. The second case isn't so easy to dismiss. + # The process group is orphaned when the first child terminates and + # POSIX.1 requires that every STOPPED process in an orphaned process + # group be sent a SIGHUP signal followed by a SIGCONT signal. Since the + # second child is not STOPPED though, we can safely forego ignoring the + # SIGHUP signal. In any case, there are no ill-effects if it is ignored. + # + # import signal # Set handlers for asynchronous events. + # signal.signal(signal.SIGHUP, signal.SIG_IGN) + + try: + # Fork a second child and exit immediately to prevent zombies. This + # causes the second child process to be orphaned, making the init + # process responsible for its cleanup. And, since the first child is + # a session leader without a controlling terminal, it's possible for + # it to acquire one by opening a terminal in the future (System V- + # based systems). This second fork guarantees that the child is no + # longer a session leader, preventing the daemon from ever acquiring + # a controlling terminal. + pid = os.fork() # Fork a second child. + except OSError as e: + raise Exception("%s [%d]" % (e.strerror, e.errno)) + + if (pid == 0): # The second child. + # We probably don't want the file mode creation mask inherited from + # the parent, so we give the child complete control over permissions. + if UMASK is not None: + os.umask(UMASK) + else: + # Parent (the first child) of the second child. + os._exit(0) + else: + # exit() or _exit()? + # _exit is like exit(), but it doesn't call any functions registered + # with atexit (and on_exit) or any registered signal handlers. It also + # closes any open file descriptors. Using exit() may cause all stdio + # streams to be flushed twice and any temporary files may be unexpectedly + # removed. It's therefore recommended that child branches of a fork() + # and the parent branch(es) of a daemon use _exit(). + return + + # Close all open file descriptors. This prevents the child from keeping + # open any file descriptors inherited from the parent. There is a variety + # of methods to accomplish this task. Three are listed below. + # + # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum + # number of open file descriptors to close. If it doesn't exist, use + # the default value (configurable). + # + # try: + # maxfd = os.sysconf("SC_OPEN_MAX") + # except (AttributeError, ValueError): + # maxfd = MAXFD + # + # OR + # + # if (os.sysconf_names.has_key("SC_OPEN_MAX")): + # maxfd = os.sysconf("SC_OPEN_MAX") + # else: + # maxfd = MAXFD + # + # OR + # + # Use the getrlimit method to retrieve the maximum file descriptor number + # that can be opened by this process. If there is no limit on the + # resource, use the default value. + # + import resource # Resource usage information. + maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + if (maxfd == resource.RLIM_INFINITY): + maxfd = MAXFD + + # Iterate through and close all file descriptors. +# for fd in range(0, maxfd): +# try: +# os.close(fd) +# except OSError: # ERROR, fd wasn't open to begin with (ignored) +# pass + + # Redirect the standard I/O file descriptors to the specified file. Since + # the daemon has no controlling terminal, most daemons redirect stdin, + # stdout, and stderr to /dev/null. This is done to prevent side-effects + # from reads and writes to the standard I/O file descriptors. + + # This call to open is guaranteed to return the lowest file descriptor, + # which will be 0 (stdin), since it was closed above. +# os.open(REDIRECT_TO, os.O_RDWR) # standard input (0) + + # Duplicate standard input to standard output and standard error. +# os.dup2(0, 1) # standard output (1) +# os.dup2(0, 2) # standard error (2) + + + si = file('/dev/null', 'r') + so = file(logfile, 'w') + se = so + + + # Replace those fds with our own + os.dup2(si.fileno(), sys.stdin.fileno()) + os.dup2(so.fileno(), sys.stdout.fileno()) + os.dup2(se.fileno(), sys.stderr.fileno()) + + function() + + os._exit(0) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/data.py b/import-layers/yocto-poky/bitbake/lib/bb/data.py new file mode 100644 index 000000000..dbc6dea68 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/data.py @@ -0,0 +1,448 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Data' implementations + +Functions for interacting with the data structure used by the +BitBake build tools. + +The expandKeys and update_data are the most expensive +operations. At night the cookie monster came by and +suggested 'give me cookies on setting the variables and +things will work out'. Taking this suggestion into account +applying the skills from the not yet passed 'Entwurf und +Analyse von Algorithmen' lecture and the cookie +monster seems to be right. We will track setVar more carefully +to have faster update_data and expandKeys operations. + +This is a trade-off between speed and memory again but +the speed is more critical here. +""" + +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2005 Holger Hans Peter Freyther +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import sys, os, re +if sys.argv[0][-5:] == "pydoc": + path = os.path.dirname(os.path.dirname(sys.argv[1])) +else: + path = os.path.dirname(os.path.dirname(sys.argv[0])) +sys.path.insert(0, path) +from itertools import groupby + +from bb import data_smart +from bb import codeparser +import bb + +logger = data_smart.logger +_dict_type = data_smart.DataSmart + +def init(): + """Return a new object representing the Bitbake data""" + return _dict_type() + +def init_db(parent = None): + """Return a new object representing the Bitbake data, + optionally based on an existing object""" + if parent is not None: + return parent.createCopy() + else: + return _dict_type() + +def createCopy(source): + """Link the source set to the destination + If one does not find the value in the destination set, + search will go on to the source set to get the value. + Value from source are copy-on-write. i.e. any try to + modify one of them will end up putting the modified value + in the destination set. + """ + return source.createCopy() + +def initVar(var, d): + """Non-destructive var init for data structure""" + d.initVar(var) + + +def setVar(var, value, d): + """Set a variable to a given value""" + d.setVar(var, value) + + +def getVar(var, d, exp = False): + """Gets the value of a variable""" + return d.getVar(var, exp) + + +def renameVar(key, newkey, d): + """Renames a variable from key to newkey""" + d.renameVar(key, newkey) + +def delVar(var, d): + """Removes a variable from the data set""" + d.delVar(var) + +def appendVar(var, value, d): + """Append additional value to a variable""" + d.appendVar(var, value) + +def setVarFlag(var, flag, flagvalue, d): + """Set a flag for a given variable to a given value""" + d.setVarFlag(var, flag, flagvalue) + +def getVarFlag(var, flag, d): + """Gets given flag from given var""" + return d.getVarFlag(var, flag, False) + +def delVarFlag(var, flag, d): + """Removes a given flag from the variable's flags""" + d.delVarFlag(var, flag) + +def setVarFlags(var, flags, d): + """Set the flags for a given variable + + Note: + setVarFlags will not clear previous + flags. Think of this method as + addVarFlags + """ + d.setVarFlags(var, flags) + +def getVarFlags(var, d): + """Gets a variable's flags""" + return d.getVarFlags(var) + +def delVarFlags(var, d): + """Removes a variable's flags""" + d.delVarFlags(var) + +def keys(d): + """Return a list of keys in d""" + return d.keys() + + +__expand_var_regexp__ = re.compile(r"\${[^{}]+}") +__expand_python_regexp__ = re.compile(r"\${@.+?}") + +def expand(s, d, varname = None): + """Variable expansion using the data store""" + return d.expand(s, varname) + +def expandKeys(alterdata, readdata = None): + if readdata == None: + readdata = alterdata + + todolist = {} + for key in alterdata: + if not '${' in key: + continue + + ekey = expand(key, readdata) + if key == ekey: + continue + todolist[key] = ekey + + # These two for loops are split for performance to maximise the + # usefulness of the expand cache + for key in sorted(todolist): + ekey = todolist[key] + newval = alterdata.getVar(ekey, False) + if newval is not None: + val = alterdata.getVar(key, False) + if val is not None: + bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval)) + alterdata.renameVar(key, ekey) + +def inheritFromOS(d, savedenv, permitted): + """Inherit variables from the initial environment.""" + exportlist = bb.utils.preserved_envvars_exported() + for s in savedenv.keys(): + if s in permitted: + try: + d.setVar(s, savedenv.getVar(s, True), op = 'from env') + if s in exportlist: + d.setVarFlag(s, "export", True, op = 'auto env export') + except TypeError: + pass + +def emit_var(var, o=sys.__stdout__, d = init(), all=False): + """Emit a variable to be sourced by a shell.""" + if d.getVarFlag(var, "python", False): + return False + + export = d.getVarFlag(var, "export", False) + unexport = d.getVarFlag(var, "unexport", False) + func = d.getVarFlag(var, "func", False) + if not all and not export and not unexport and not func: + return False + + try: + if all: + oval = d.getVar(var, False) + val = d.getVar(var, True) + except (KeyboardInterrupt, bb.build.FuncFailed): + raise + except Exception as exc: + o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc))) + return False + + if all: + d.varhistory.emit(var, oval, val, o, d) + + if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all: + return False + + varExpanded = d.expand(var) + + if unexport: + o.write('unset %s\n' % varExpanded) + return False + + if val is None: + return False + + val = str(val) + + if varExpanded.startswith("BASH_FUNC_"): + varExpanded = varExpanded[10:-2] + val = val[3:] # Strip off "() " + o.write("%s() %s\n" % (varExpanded, val)) + o.write("export -f %s\n" % (varExpanded)) + return True + + if func: + # NOTE: should probably check for unbalanced {} within the var + val = val.rstrip('\n') + o.write("%s() {\n%s\n}\n" % (varExpanded, val)) + return 1 + + if export: + o.write('export ') + + # if we're going to output this within doublequotes, + # to a shell, we need to escape the quotes in the var + alter = re.sub('"', '\\"', val) + alter = re.sub('\n', ' \\\n', alter) + alter = re.sub('\\$', '\\\\$', alter) + o.write('%s="%s"\n' % (varExpanded, alter)) + return False + +def emit_env(o=sys.__stdout__, d = init(), all=False): + """Emits all items in the data store in a format such that it can be sourced by a shell.""" + + isfunc = lambda key: bool(d.getVarFlag(key, "func", False)) + keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc) + grouped = groupby(keys, isfunc) + for isfunc, keys in grouped: + for key in keys: + emit_var(key, o, d, all and not isfunc) and o.write('\n') + +def exported_keys(d): + return (key for key in d.keys() if not key.startswith('__') and + d.getVarFlag(key, 'export', False) and + not d.getVarFlag(key, 'unexport', False)) + +def exported_vars(d): + for key in exported_keys(d): + try: + value = d.getVar(key, True) + except Exception: + pass + + if value is not None: + yield key, str(value) + +def emit_func(func, o=sys.__stdout__, d = init()): + """Emits all items in the data store in a format such that it can be sourced by a shell.""" + + keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func", False)) + for key in keys: + emit_var(key, o, d, False) + + o.write('\n') + emit_var(func, o, d, False) and o.write('\n') + newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True)) + newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split()) + seen = set() + while newdeps: + deps = newdeps + seen |= deps + newdeps = set() + for dep in deps: + if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False): + emit_var(dep, o, d, False) and o.write('\n') + newdeps |= bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep, True)) + newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split()) + newdeps -= seen + +_functionfmt = """ +def {function}(d): +{body}""" + +def emit_func_python(func, o=sys.__stdout__, d = init()): + """Emits all items in the data store in a format such that it can be sourced by a shell.""" + + def write_func(func, o, call = False): + body = d.getVar(func, False) + if not body.startswith("def"): + body = _functionfmt.format(function=func, body=body) + + o.write(body.strip() + "\n\n") + if call: + o.write(func + "(d)" + "\n\n") + + write_func(func, o, True) + pp = bb.codeparser.PythonParser(func, logger) + pp.parse_python(d.getVar(func, False)) + newdeps = pp.execs + newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split()) + seen = set() + while newdeps: + deps = newdeps + seen |= deps + newdeps = set() + for dep in deps: + if d.getVarFlag(dep, "func", False) and d.getVarFlag(dep, "python", False): + write_func(dep, o) + pp = bb.codeparser.PythonParser(dep, logger) + pp.parse_python(d.getVar(dep, False)) + newdeps |= pp.execs + newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split()) + newdeps -= seen + +def update_data(d): + """Performs final steps upon the datastore, including application of overrides""" + d.finalize(parent = True) + +def build_dependencies(key, keys, shelldeps, varflagsexcl, d): + deps = set() + try: + if key[-1] == ']': + vf = key[:-1].split('[') + value = d.getVarFlag(vf[0], vf[1], False) + parser = d.expandWithRefs(value, key) + deps |= parser.references + deps = deps | (keys & parser.execs) + return deps, value + varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "vardepvalueexclude", "postfuncs", "prefuncs", "lineno", "filename"]) or {} + vardeps = varflags.get("vardeps") + value = d.getVar(key, False) + + def handle_contains(value, contains, d): + newvalue = "" + for k in sorted(contains): + l = (d.getVar(k, True) or "").split() + for word in sorted(contains[k]): + if word in l: + newvalue += "\n%s{%s} = Set" % (k, word) + else: + newvalue += "\n%s{%s} = Unset" % (k, word) + if not newvalue: + return value + if not value: + return newvalue + return value + newvalue + + if "vardepvalue" in varflags: + value = varflags.get("vardepvalue") + elif varflags.get("func"): + if varflags.get("python"): + parser = bb.codeparser.PythonParser(key, logger) + if value and "\t" in value: + logger.warn("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE", True))) + parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno")) + deps = deps | parser.references + deps = deps | (keys & parser.execs) + value = handle_contains(value, parser.contains, d) + else: + parsedvar = d.expandWithRefs(value, key) + parser = bb.codeparser.ShellParser(key, logger) + parser.parse_shell(parsedvar.value) + deps = deps | shelldeps + deps = deps | parsedvar.references + deps = deps | (keys & parser.execs) | (keys & parsedvar.execs) + value = handle_contains(value, parsedvar.contains, d) + if vardeps is None: + parser.log.flush() + if "prefuncs" in varflags: + deps = deps | set(varflags["prefuncs"].split()) + if "postfuncs" in varflags: + deps = deps | set(varflags["postfuncs"].split()) + else: + parser = d.expandWithRefs(value, key) + deps |= parser.references + deps = deps | (keys & parser.execs) + value = handle_contains(value, parser.contains, d) + + if "vardepvalueexclude" in varflags: + exclude = varflags.get("vardepvalueexclude") + for excl in exclude.split('|'): + if excl: + value = value.replace(excl, '') + + # Add varflags, assuming an exclusion list is set + if varflagsexcl: + varfdeps = [] + for f in varflags: + if f not in varflagsexcl: + varfdeps.append('%s[%s]' % (key, f)) + if varfdeps: + deps |= set(varfdeps) + + deps |= set((vardeps or "").split()) + deps -= set(varflags.get("vardepsexclude", "").split()) + except Exception as e: + bb.warn("Exception during build_dependencies for %s" % key) + raise + return deps, value + #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) + #d.setVarFlag(key, "vardeps", deps) + +def generate_dependencies(d): + + keys = set(key for key in d if not key.startswith("__")) + shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) + varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS', True) + + deps = {} + values = {} + + tasklist = d.getVar('__BBTASKS', False) or [] + for task in tasklist: + deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d) + newdeps = deps[task] + seen = set() + while newdeps: + nextdeps = newdeps + seen |= nextdeps + newdeps = set() + for dep in nextdeps: + if dep not in deps: + deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d) + newdeps |= deps[dep] + newdeps -= seen + #print "For %s: %s" % (task, str(deps[task])) + return tasklist, deps, values + +def inherits_class(klass, d): + val = d.getVar('__inherit_cache', False) or [] + needle = os.path.join('classes', '%s.bbclass' % klass) + for v in val: + if v.endswith(needle): + return True + return False diff --git a/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py b/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py new file mode 100644 index 000000000..fa1e79427 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py @@ -0,0 +1,969 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake Smart Dictionary Implementation + +Functions for interacting with the data structure used by the +BitBake build tools. + +""" + +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2004, 2005 Seb Frankengul +# Copyright (C) 2005, 2006 Holger Hans Peter Freyther +# Copyright (C) 2005 Uli Luckas +# Copyright (C) 2005 ROAD GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import copy, re, sys, traceback +from collections import MutableMapping +import logging +import hashlib +import bb, bb.codeparser +from bb import utils +from bb.COW import COWDictBase + +logger = logging.getLogger("BitBake.Data") + +__setvar_keyword__ = ["_append", "_prepend", "_remove"] +__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>.*))?$') +__expand_var_regexp__ = re.compile(r"\${[^{}@\n\t :]+}") +__expand_python_regexp__ = re.compile(r"\${@.+?}") + +def infer_caller_details(loginfo, parent = False, varval = True): + """Save the caller the trouble of specifying everything.""" + # Save effort. + if 'ignore' in loginfo and loginfo['ignore']: + return + # If nothing was provided, mark this as possibly unneeded. + if not loginfo: + loginfo['ignore'] = True + return + # Infer caller's likely values for variable (var) and value (value), + # to reduce clutter in the rest of the code. + above = None + def set_above(): + try: + raise Exception + except Exception: + tb = sys.exc_info()[2] + if parent: + return tb.tb_frame.f_back.f_back.f_back + else: + return tb.tb_frame.f_back.f_back + + if varval and ('variable' not in loginfo or 'detail' not in loginfo): + if not above: + above = set_above() + lcls = above.f_locals.items() + for k, v in lcls: + if k == 'value' and 'detail' not in loginfo: + loginfo['detail'] = v + if k == 'var' and 'variable' not in loginfo: + loginfo['variable'] = v + # Infer file/line/function from traceback + # Don't use traceback.extract_stack() since it fills the line contents which + # we don't need and that hits stat syscalls + if 'file' not in loginfo: + if not above: + above = set_above() + f = above.f_back + line = f.f_lineno + file = f.f_code.co_filename + func = f.f_code.co_name + loginfo['file'] = file + loginfo['line'] = line + if func not in loginfo: + loginfo['func'] = func + +class VariableParse: + def __init__(self, varname, d, val = None): + self.varname = varname + self.d = d + self.value = val + + self.references = set() + self.execs = set() + self.contains = {} + + def var_sub(self, match): + key = match.group()[2:-1] + if self.varname and key: + if self.varname == key: + raise Exception("variable %s references itself!" % self.varname) + if key in self.d.expand_cache: + varparse = self.d.expand_cache[key] + var = varparse.value + else: + var = self.d.getVarFlag(key, "_content", True) + self.references.add(key) + if var is not None: + return var + else: + return match.group() + + def python_sub(self, match): + code = match.group()[3:-1] + codeobj = compile(code.strip(), self.varname or "<expansion>", "eval") + + parser = bb.codeparser.PythonParser(self.varname, logger) + parser.parse_python(code) + if self.varname: + vardeps = self.d.getVarFlag(self.varname, "vardeps", True) + if vardeps is None: + parser.log.flush() + else: + parser.log.flush() + self.references |= parser.references + self.execs |= parser.execs + + for k in parser.contains: + if k not in self.contains: + self.contains[k] = parser.contains[k].copy() + else: + self.contains[k].update(parser.contains[k]) + value = utils.better_eval(codeobj, DataContext(self.d)) + return str(value) + + +class DataContext(dict): + def __init__(self, metadata, **kwargs): + self.metadata = metadata + dict.__init__(self, **kwargs) + self['d'] = metadata + + def __missing__(self, key): + value = self.metadata.getVar(key, True) + if value is None or self.metadata.getVarFlag(key, 'func', False): + raise KeyError(key) + else: + return value + +class ExpansionError(Exception): + def __init__(self, varname, expression, exception): + self.expression = expression + self.variablename = varname + self.exception = exception + if varname: + if expression: + self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) + else: + self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) + else: + self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) + Exception.__init__(self, self.msg) + self.args = (varname, expression, exception) + def __str__(self): + return self.msg + +class IncludeHistory(object): + def __init__(self, parent = None, filename = '[TOP LEVEL]'): + self.parent = parent + self.filename = filename + self.children = [] + self.current = self + + def copy(self): + new = IncludeHistory(self.parent, self.filename) + for c in self.children: + new.children.append(c) + return new + + def include(self, filename): + newfile = IncludeHistory(self.current, filename) + self.current.children.append(newfile) + self.current = newfile + return self + + def __enter__(self): + pass + + def __exit__(self, a, b, c): + if self.current.parent: + self.current = self.current.parent + else: + bb.warn("Include log: Tried to finish '%s' at top level." % filename) + return False + + def emit(self, o, level = 0): + """Emit an include history file, and its children.""" + if level: + spaces = " " * (level - 1) + o.write("# %s%s" % (spaces, self.filename)) + if len(self.children) > 0: + o.write(" includes:") + else: + o.write("#\n# INCLUDE HISTORY:\n#") + level = level + 1 + for child in self.children: + o.write("\n") + child.emit(o, level) + +class VariableHistory(object): + def __init__(self, dataroot): + self.dataroot = dataroot + self.variables = COWDictBase.copy() + + def copy(self): + new = VariableHistory(self.dataroot) + new.variables = self.variables.copy() + return new + + def record(self, *kwonly, **loginfo): + if not self.dataroot._tracking: + return + if len(kwonly) > 0: + raise TypeError + infer_caller_details(loginfo, parent = True) + if 'ignore' in loginfo and loginfo['ignore']: + return + if 'op' not in loginfo or not loginfo['op']: + loginfo['op'] = 'set' + if 'detail' in loginfo: + loginfo['detail'] = str(loginfo['detail']) + if 'variable' not in loginfo or 'file' not in loginfo: + raise ValueError("record() missing variable or file.") + var = loginfo['variable'] + + if var not in self.variables: + self.variables[var] = [] + if not isinstance(self.variables[var], list): + return + if 'nodups' in loginfo and loginfo in self.variables[var]: + return + self.variables[var].append(loginfo.copy()) + + def variable(self, var): + if var in self.variables: + return self.variables[var] + else: + return [] + + def emit(self, var, oval, val, o, d): + history = self.variable(var) + + # Append override history + if var in d.overridedata: + for (r, override) in d.overridedata[var]: + for event in self.variable(r): + loginfo = event.copy() + if 'flag' in loginfo and not loginfo['flag'].startswith("_"): + continue + loginfo['variable'] = var + loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) + history.append(loginfo) + + commentVal = re.sub('\n', '\n#', str(oval)) + if history: + if len(history) == 1: + o.write("#\n# $%s\n" % var) + else: + o.write("#\n# $%s [%d operations]\n" % (var, len(history))) + for event in history: + # o.write("# %s\n" % str(event)) + if 'func' in event: + # If we have a function listed, this is internal + # code, not an operation in a config file, and the + # full path is distracting. + event['file'] = re.sub('.*/', '', event['file']) + display_func = ' [%s]' % event['func'] + else: + display_func = '' + if 'flag' in event: + flag = '[%s] ' % (event['flag']) + else: + flag = '' + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) + if len(history) > 1: + o.write("# pre-expansion value:\n") + o.write('# "%s"\n' % (commentVal)) + else: + o.write("#\n# $%s\n# [no history recorded]\n#\n" % var) + o.write('# "%s"\n' % (commentVal)) + + def get_variable_files(self, var): + """Get the files where operations are made on a variable""" + var_history = self.variable(var) + files = [] + for event in var_history: + files.append(event['file']) + return files + + def get_variable_lines(self, var, f): + """Get the line where a operation is made on a variable in file f""" + var_history = self.variable(var) + lines = [] + for event in var_history: + if f== event['file']: + line = event['line'] + lines.append(line) + return lines + + def get_variable_items_files(self, var, d): + """ + Use variable history to map items added to a list variable and + the files in which they were added. + """ + history = self.variable(var) + finalitems = (d.getVar(var, True) or '').split() + filemap = {} + isset = False + for event in history: + if 'flag' in event: + continue + if event['op'] == '_remove': + continue + if isset and event['op'] == 'set?': + continue + isset = True + items = d.expand(event['detail']).split() + for item in items: + # This is a little crude but is belt-and-braces to avoid us + # having to handle every possible operation type specifically + if item in finalitems and not item in filemap: + filemap[item] = event['file'] + return filemap + + def del_var_history(self, var, f=None, line=None): + """If file f and line are not given, the entire history of var is deleted""" + if var in self.variables: + if f and line: + self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] + else: + self.variables[var] = [] + +class DataSmart(MutableMapping): + def __init__(self): + self.dict = {} + + self.inchistory = IncludeHistory() + self.varhistory = VariableHistory(self) + self._tracking = False + + self.expand_cache = {} + + # cookie monster tribute + # Need to be careful about writes to overridedata as + # its only a shallow copy, could influence other data store + # copies! + self.overridedata = {} + self.overrides = None + self.overridevars = set(["OVERRIDES", "FILE"]) + self.inoverride = False + + def enableTracking(self): + self._tracking = True + + def disableTracking(self): + self._tracking = False + + def expandWithRefs(self, s, varname): + + if not isinstance(s, basestring): # sanity check + return VariableParse(varname, self, s) + + if varname and varname in self.expand_cache: + return self.expand_cache[varname] + + varparse = VariableParse(varname, self) + + while s.find('${') != -1: + olds = s + try: + s = __expand_var_regexp__.sub(varparse.var_sub, s) + try: + s = __expand_python_regexp__.sub(varparse.python_sub, s) + except SyntaxError as e: + # Likely unmatched brackets, just don't expand the expression + if e.msg != "EOL while scanning string literal": + raise + if s == olds: + break + except ExpansionError: + raise + except bb.parse.SkipRecipe: + raise + except Exception as exc: + exc_class, exc, tb = sys.exc_info() + raise ExpansionError, ExpansionError(varname, s, exc), tb + + varparse.value = s + + if varname: + self.expand_cache[varname] = varparse + + return varparse + + def expand(self, s, varname = None): + return self.expandWithRefs(s, varname).value + + def finalize(self, parent = False): + return + + def internal_finalize(self, parent = False): + """Performs final steps upon the datastore, including application of overrides""" + self.overrides = None + + def need_overrides(self): + if self.overrides is not None: + return + if self.inoverride: + return + for count in range(5): + self.inoverride = True + # Can end up here recursively so setup dummy values + self.overrides = [] + self.overridesset = set() + self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + self.overridesset = set(self.overrides) + self.inoverride = False + self.expand_cache = {} + newoverrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + if newoverrides == self.overrides: + break + self.overrides = newoverrides + self.overridesset = set(self.overrides) + else: + bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.") + + def initVar(self, var): + self.expand_cache = {} + if not var in self.dict: + self.dict[var] = {} + + def _findVar(self, var): + dest = self.dict + while dest: + if var in dest: + return dest[var] + + if "_data" not in dest: + break + dest = dest["_data"] + + def _makeShadowCopy(self, var): + if var in self.dict: + return + + local_var = self._findVar(var) + + if local_var: + self.dict[var] = copy.copy(local_var) + else: + self.initVar(var) + + + def setVar(self, var, value, **loginfo): + #print("var=" + str(var) + " val=" + str(value)) + parsing=False + if 'parsing' in loginfo: + parsing=True + + if 'op' not in loginfo: + loginfo['op'] = "set" + self.expand_cache = {} + match = __setvar_regexp__.match(var) + if match and match.group("keyword") in __setvar_keyword__: + base = match.group('base') + keyword = match.group("keyword") + override = match.group('add') + l = self.getVarFlag(base, keyword, False) or [] + l.append([value, override]) + self.setVarFlag(base, keyword, l, ignore=True) + # And cause that to be recorded: + loginfo['detail'] = value + loginfo['variable'] = base + if override: + loginfo['op'] = '%s[%s]' % (keyword, override) + else: + loginfo['op'] = keyword + self.varhistory.record(**loginfo) + # todo make sure keyword is not __doc__ or __module__ + # pay the cookie monster + + # more cookies for the cookie monster + if '_' in var: + self._setvar_update_overrides(base, **loginfo) + + if base in self.overridevars: + self._setvar_update_overridevars(var, value) + return + + if not var in self.dict: + self._makeShadowCopy(var) + + if not parsing: + if "_append" in self.dict[var]: + del self.dict[var]["_append"] + if "_prepend" in self.dict[var]: + del self.dict[var]["_prepend"] + if var in self.overridedata: + active = [] + self.need_overrides() + for (r, o) in self.overridedata[var]: + if o in self.overridesset: + active.append(r) + elif "_" in o: + if set(o.split("_")).issubset(self.overridesset): + active.append(r) + for a in active: + self.delVar(a) + del self.overridedata[var] + + # more cookies for the cookie monster + if '_' in var: + self._setvar_update_overrides(var, **loginfo) + + # setting var + self.dict[var]["_content"] = value + self.varhistory.record(**loginfo) + + if var in self.overridevars: + self._setvar_update_overridevars(var, value) + + def _setvar_update_overridevars(self, var, value): + vardata = self.expandWithRefs(value, var) + new = vardata.references + new.update(vardata.contains.keys()) + while not new.issubset(self.overridevars): + nextnew = set() + self.overridevars.update(new) + for i in new: + vardata = self.expandWithRefs(self.getVar(i, True), i) + nextnew.update(vardata.references) + nextnew.update(vardata.contains.keys()) + new = nextnew + self.internal_finalize(True) + + def _setvar_update_overrides(self, var, **loginfo): + # aka pay the cookie monster + override = var[var.rfind('_')+1:] + shortvar = var[:var.rfind('_')] + while override and override.islower(): + if shortvar not in self.overridedata: + self.overridedata[shortvar] = [] + if [var, override] not in self.overridedata[shortvar]: + # Force CoW by recreating the list first + self.overridedata[shortvar] = list(self.overridedata[shortvar]) + self.overridedata[shortvar].append([var, override]) + override = None + if "_" in shortvar: + override = var[shortvar.rfind('_')+1:] + shortvar = var[:shortvar.rfind('_')] + if len(shortvar) == 0: + override = None + + def getVar(self, var, expand, noweakdefault=False, parsing=False): + return self.getVarFlag(var, "_content", expand, noweakdefault, parsing) + + def renameVar(self, key, newkey, **loginfo): + """ + Rename the variable key to newkey + """ + val = self.getVar(key, 0, parsing=True) + if val is not None: + loginfo['variable'] = newkey + loginfo['op'] = 'rename from %s' % key + loginfo['detail'] = val + self.varhistory.record(**loginfo) + self.setVar(newkey, val, ignore=True, parsing=True) + + for i in (__setvar_keyword__): + src = self.getVarFlag(key, i, False) + if src is None: + continue + + dest = self.getVarFlag(newkey, i, False) or [] + dest.extend(src) + self.setVarFlag(newkey, i, dest, ignore=True) + + if key in self.overridedata: + self.overridedata[newkey] = [] + for (v, o) in self.overridedata[key]: + self.overridedata[newkey].append([v.replace(key, newkey), o]) + self.renameVar(v, v.replace(key, newkey)) + + if '_' in newkey and val is None: + self._setvar_update_overrides(newkey, **loginfo) + + loginfo['variable'] = key + loginfo['op'] = 'rename (to)' + loginfo['detail'] = newkey + self.varhistory.record(**loginfo) + self.delVar(key, ignore=True) + + def appendVar(self, var, value, **loginfo): + loginfo['op'] = 'append' + self.varhistory.record(**loginfo) + self.setVar(var + "_append", value, ignore=True, parsing=True) + + def prependVar(self, var, value, **loginfo): + loginfo['op'] = 'prepend' + self.varhistory.record(**loginfo) + self.setVar(var + "_prepend", value, ignore=True, parsing=True) + + def delVar(self, var, **loginfo): + loginfo['detail'] = "" + loginfo['op'] = 'del' + self.varhistory.record(**loginfo) + self.expand_cache = {} + self.dict[var] = {} + if var in self.overridedata: + del self.overridedata[var] + if '_' in var: + override = var[var.rfind('_')+1:] + shortvar = var[:var.rfind('_')] + while override and override.islower(): + try: + if shortvar in self.overridedata: + # Force CoW by recreating the list first + self.overridedata[shortvar] = list(self.overridedata[shortvar]) + self.overridedata[shortvar].remove([var, override]) + except ValueError as e: + pass + override = None + if "_" in shortvar: + override = var[shortvar.rfind('_')+1:] + shortvar = var[:shortvar.rfind('_')] + if len(shortvar) == 0: + override = None + + def setVarFlag(self, var, flag, value, **loginfo): + self.expand_cache = {} + if 'op' not in loginfo: + loginfo['op'] = "set" + loginfo['flag'] = flag + self.varhistory.record(**loginfo) + if not var in self.dict: + self._makeShadowCopy(var) + self.dict[var][flag] = value + + if flag == "_defaultval" and '_' in var: + self._setvar_update_overrides(var, **loginfo) + if flag == "_defaultval" and var in self.overridevars: + self._setvar_update_overridevars(var, value) + + if flag == "unexport" or flag == "export": + if not "__exportlist" in self.dict: + self._makeShadowCopy("__exportlist") + if not "_content" in self.dict["__exportlist"]: + self.dict["__exportlist"]["_content"] = set() + self.dict["__exportlist"]["_content"].add(var) + + def getVarFlag(self, var, flag, expand, noweakdefault=False, parsing=False): + local_var = self._findVar(var) + value = None + if flag == "_content" and var in self.overridedata and not parsing: + match = False + active = {} + self.need_overrides() + for (r, o) in self.overridedata[var]: + # What about double overrides both with "_" in the name? + if o in self.overridesset: + active[o] = r + elif "_" in o: + if set(o.split("_")).issubset(self.overridesset): + active[o] = r + + mod = True + while mod: + mod = False + for o in self.overrides: + for a in active.copy(): + if a.endswith("_" + o): + t = active[a] + del active[a] + active[a.replace("_" + o, "")] = t + mod = True + elif a == o: + match = active[a] + del active[a] + if match: + value = self.getVar(match, False) + + if local_var is not None and value is None: + if flag in local_var: + value = copy.copy(local_var[flag]) + elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: + value = copy.copy(local_var["_defaultval"]) + + + if flag == "_content" and local_var is not None and "_append" in local_var and not parsing: + if not value: + value = "" + self.need_overrides() + for (r, o) in local_var["_append"]: + match = True + if o: + for o2 in o.split("_"): + if not o2 in self.overrides: + match = False + if match: + value = value + r + + if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing: + if not value: + value = "" + self.need_overrides() + for (r, o) in local_var["_prepend"]: + + match = True + if o: + for o2 in o.split("_"): + if not o2 in self.overrides: + match = False + if match: + value = r + value + + if expand and value: + # Only getvar (flag == _content) hits the expand cache + cachename = None + if flag == "_content": + cachename = var + else: + cachename = var + "[" + flag + "]" + value = self.expand(value, cachename) + + if value and flag == "_content" and local_var is not None and "_remove" in local_var: + removes = [] + self.need_overrides() + for (r, o) in local_var["_remove"]: + match = True + if o: + for o2 in o.split("_"): + if not o2 in self.overrides: + match = False + if match: + removes.extend(self.expand(r).split()) + + filtered = filter(lambda v: v not in removes, + value.split()) + value = " ".join(filtered) + if expand and var in self.expand_cache: + # We need to ensure the expand cache has the correct value + # flag == "_content" here + self.expand_cache[var].value = value + return value + + def delVarFlag(self, var, flag, **loginfo): + self.expand_cache = {} + local_var = self._findVar(var) + if not local_var: + return + if not var in self.dict: + self._makeShadowCopy(var) + + if var in self.dict and flag in self.dict[var]: + loginfo['detail'] = "" + loginfo['op'] = 'delFlag' + loginfo['flag'] = flag + self.varhistory.record(**loginfo) + + del self.dict[var][flag] + + def appendVarFlag(self, var, flag, value, **loginfo): + loginfo['op'] = 'append' + loginfo['flag'] = flag + self.varhistory.record(**loginfo) + newvalue = (self.getVarFlag(var, flag, False) or "") + value + self.setVarFlag(var, flag, newvalue, ignore=True) + + def prependVarFlag(self, var, flag, value, **loginfo): + loginfo['op'] = 'prepend' + loginfo['flag'] = flag + self.varhistory.record(**loginfo) + newvalue = value + (self.getVarFlag(var, flag, False) or "") + self.setVarFlag(var, flag, newvalue, ignore=True) + + def setVarFlags(self, var, flags, **loginfo): + self.expand_cache = {} + infer_caller_details(loginfo) + if not var in self.dict: + self._makeShadowCopy(var) + + for i in flags: + if i == "_content": + continue + loginfo['flag'] = i + loginfo['detail'] = flags[i] + self.varhistory.record(**loginfo) + self.dict[var][i] = flags[i] + + def getVarFlags(self, var, expand = False, internalflags=False): + local_var = self._findVar(var) + flags = {} + + if local_var: + for i in local_var: + if i.startswith("_") and not internalflags: + continue + flags[i] = local_var[i] + if expand and i in expand: + flags[i] = self.expand(flags[i], var + "[" + i + "]") + if len(flags) == 0: + return None + return flags + + + def delVarFlags(self, var, **loginfo): + self.expand_cache = {} + if not var in self.dict: + self._makeShadowCopy(var) + + if var in self.dict: + content = None + + loginfo['op'] = 'delete flags' + self.varhistory.record(**loginfo) + + # try to save the content + if "_content" in self.dict[var]: + content = self.dict[var]["_content"] + self.dict[var] = {} + self.dict[var]["_content"] = content + else: + del self.dict[var] + + def createCopy(self): + """ + Create a copy of self by setting _data to self + """ + # we really want this to be a DataSmart... + data = DataSmart() + data.dict["_data"] = self.dict + data.varhistory = self.varhistory.copy() + data.varhistory.datasmart = data + data.inchistory = self.inchistory.copy() + + data._tracking = self._tracking + + data.overrides = None + data.overridevars = copy.copy(self.overridevars) + # Should really be a deepcopy but has heavy overhead. + # Instead, we're careful with writes. + data.overridedata = copy.copy(self.overridedata) + + return data + + def expandVarref(self, variable, parents=False): + """Find all references to variable in the data and expand it + in place, optionally descending to parent datastores.""" + + if parents: + keys = iter(self) + else: + keys = self.localkeys() + + ref = '${%s}' % variable + value = self.getVar(variable, False) + for key in keys: + referrervalue = self.getVar(key, False) + if referrervalue and ref in referrervalue: + self.setVar(key, referrervalue.replace(ref, value)) + + def localkeys(self): + for key in self.dict: + if key != '_data': + yield key + + def __iter__(self): + deleted = set() + overrides = set() + def keylist(d): + klist = set() + for key in d: + if key == "_data": + continue + if key in deleted: + continue + if key in overrides: + continue + if not d[key]: + deleted.add(key) + continue + klist.add(key) + + if "_data" in d: + klist |= keylist(d["_data"]) + + return klist + + self.need_overrides() + for var in self.overridedata: + for (r, o) in self.overridedata[var]: + if o in self.overridesset: + overrides.add(var) + elif "_" in o: + if set(o.split("_")).issubset(self.overridesset): + overrides.add(var) + + for k in keylist(self.dict): + yield k + + for k in overrides: + yield k + + def __len__(self): + return len(frozenset(self)) + + def __getitem__(self, item): + value = self.getVar(item, False) + if value is None: + raise KeyError(item) + else: + return value + + def __setitem__(self, var, value): + self.setVar(var, value) + + def __delitem__(self, var): + self.delVar(var) + + def get_hash(self): + data = {} + d = self.createCopy() + bb.data.expandKeys(d) + bb.data.update_data(d) + + config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split()) + keys = set(key for key in iter(d) if not key.startswith("__")) + for key in keys: + if key in config_whitelist: + continue + + value = d.getVar(key, False) or "" + data.update({key:value}) + + varflags = d.getVarFlags(key, internalflags = True) + if not varflags: + continue + for f in varflags: + if f == "_content": + continue + data.update({'%s[%s]' % (key, f):varflags[f]}) + + for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: + bb_list = d.getVar(key, False) or [] + bb_list.sort() + data.update({key:str(bb_list)}) + + if key == "__BBANONFUNCS": + for i in bb_list: + value = d.getVar(i, False) or "" + data.update({i:value}) + + data_str = str([(k, data[k]) for k in sorted(data.keys())]) + return hashlib.md5(data_str).hexdigest() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/event.py b/import-layers/yocto-poky/bitbake/lib/bb/event.py new file mode 100644 index 000000000..5ffe89eae --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/event.py @@ -0,0 +1,679 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Event' implementation + +Classes and functions for manipulating 'events' in the +BitBake build tools. +""" + +# Copyright (C) 2003, 2004 Chris Larson +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os, sys +import warnings +try: + import cPickle as pickle +except ImportError: + import pickle +import logging +import atexit +import traceback +import ast +import bb.utils +import bb.compat +import bb.exceptions + +# This is the pid for which we should generate the event. This is set when +# the runqueue forks off. +worker_pid = 0 +worker_fire = None + +logger = logging.getLogger('BitBake.Event') + +class Event(object): + """Base class for events""" + + def __init__(self): + self.pid = worker_pid + +Registered = 10 +AlreadyRegistered = 14 + +def get_class_handlers(): + return _handlers + +def set_class_handlers(h): + global _handlers + _handlers = h + +def clean_class_handlers(): + return bb.compat.OrderedDict() + +# Internal +_handlers = clean_class_handlers() +_ui_handlers = {} +_ui_logfilters = {} +_ui_handler_seq = 0 +_event_handler_map = {} +_catchall_handlers = {} +_eventfilter = None +_uiready = False + +def execute_handler(name, handler, event, d): + event.data = d + addedd = False + if 'd' not in __builtins__: + __builtins__['d'] = d + addedd = True + try: + ret = handler(event) + except (bb.parse.SkipRecipe, bb.BBHandledException): + raise + except Exception: + etype, value, tb = sys.exc_info() + logger.error("Execution of event handler '%s' failed" % name, + exc_info=(etype, value, tb.tb_next)) + raise + except SystemExit as exc: + if exc.code != 0: + logger.error("Execution of event handler '%s' failed" % name) + raise + finally: + del event.data + if addedd: + del __builtins__['d'] + +def fire_class_handlers(event, d): + if isinstance(event, logging.LogRecord): + return + + eid = str(event.__class__)[8:-2] + evt_hmap = _event_handler_map.get(eid, {}) + for name, handler in _handlers.iteritems(): + if name in _catchall_handlers or name in evt_hmap: + if _eventfilter: + if not _eventfilter(name, handler, event, d): + continue + execute_handler(name, handler, event, d) + +ui_queue = [] +@atexit.register +def print_ui_queue(): + """If we're exiting before a UI has been spawned, display any queued + LogRecords to the console.""" + logger = logging.getLogger("BitBake") + if not _uiready: + from bb.msg import BBLogFormatter + console = logging.StreamHandler(sys.stdout) + console.setFormatter(BBLogFormatter("%(levelname)s: %(message)s")) + logger.handlers = [console] + + # First check to see if we have any proper messages + msgprint = False + for event in ui_queue: + if isinstance(event, logging.LogRecord): + if event.levelno > logging.DEBUG: + logger.handle(event) + msgprint = True + if msgprint: + return + + # Nope, so just print all of the messages we have (including debug messages) + for event in ui_queue: + if isinstance(event, logging.LogRecord): + logger.handle(event) + +def fire_ui_handlers(event, d): + if not _uiready: + # No UI handlers registered yet, queue up the messages + ui_queue.append(event) + return + + errors = [] + for h in _ui_handlers: + #print "Sending event %s" % event + try: + if not _ui_logfilters[h].filter(event): + continue + # We use pickle here since it better handles object instances + # which xmlrpc's marshaller does not. Events *must* be serializable + # by pickle. + if hasattr(_ui_handlers[h].event, "sendpickle"): + _ui_handlers[h].event.sendpickle((pickle.dumps(event))) + else: + _ui_handlers[h].event.send(event) + except: + errors.append(h) + for h in errors: + del _ui_handlers[h] + +def fire(event, d): + """Fire off an Event""" + + # We can fire class handlers in the worker process context and this is + # desired so they get the task based datastore. + # UI handlers need to be fired in the server context so we defer this. They + # don't have a datastore so the datastore context isn't a problem. + + fire_class_handlers(event, d) + if worker_fire: + worker_fire(event, d) + else: + fire_ui_handlers(event, d) + +def fire_from_worker(event, d): + fire_ui_handlers(event, d) + +noop = lambda _: None +def register(name, handler, mask=None, filename=None, lineno=None): + """Register an Event handler""" + + # already registered + if name in _handlers: + return AlreadyRegistered + + if handler is not None: + # handle string containing python code + if isinstance(handler, basestring): + tmp = "def %s(e):\n%s" % (name, handler) + try: + code = bb.methodpool.compile_cache(tmp) + if not code: + if filename is None: + filename = "%s(e)" % name + code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST) + if lineno is not None: + ast.increment_lineno(code, lineno-1) + code = compile(code, filename, "exec") + bb.methodpool.compile_cache_add(tmp, code) + except SyntaxError: + logger.error("Unable to register event handler '%s':\n%s", name, + ''.join(traceback.format_exc(limit=0))) + _handlers[name] = noop + return + env = {} + bb.utils.better_exec(code, env) + func = bb.utils.better_eval(name, env) + _handlers[name] = func + else: + _handlers[name] = handler + + if not mask or '*' in mask: + _catchall_handlers[name] = True + else: + for m in mask: + if _event_handler_map.get(m, None) is None: + _event_handler_map[m] = {} + _event_handler_map[m][name] = True + + return Registered + +def remove(name, handler): + """Remove an Event handler""" + _handlers.pop(name) + +def set_eventfilter(func): + global _eventfilter + _eventfilter = func + +def register_UIHhandler(handler, mainui=False): + if mainui: + global _uiready + _uiready = True + bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1 + _ui_handlers[_ui_handler_seq] = handler + level, debug_domains = bb.msg.constructLogOptions() + _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains) + return _ui_handler_seq + +def unregister_UIHhandler(handlerNum): + if handlerNum in _ui_handlers: + del _ui_handlers[handlerNum] + return + +# Class to allow filtering of events and specific filtering of LogRecords *before* we put them over the IPC +class UIEventFilter(object): + def __init__(self, level, debug_domains): + self.update(None, level, debug_domains) + + def update(self, eventmask, level, debug_domains): + self.eventmask = eventmask + self.stdlevel = level + self.debug_domains = debug_domains + + def filter(self, event): + if isinstance(event, logging.LogRecord): + if event.levelno >= self.stdlevel: + return True + if event.name in self.debug_domains and event.levelno >= self.debug_domains[event.name]: + return True + return False + eid = str(event.__class__)[8:-2] + if self.eventmask and eid not in self.eventmask: + return False + return True + +def set_UIHmask(handlerNum, level, debug_domains, mask): + if not handlerNum in _ui_handlers: + return False + if '*' in mask: + _ui_logfilters[handlerNum].update(None, level, debug_domains) + else: + _ui_logfilters[handlerNum].update(mask, level, debug_domains) + return True + +def getName(e): + """Returns the name of a class or class instance""" + if getattr(e, "__name__", None) == None: + return e.__class__.__name__ + else: + return e.__name__ + +class OperationStarted(Event): + """An operation has begun""" + def __init__(self, msg = "Operation Started"): + Event.__init__(self) + self.msg = msg + +class OperationCompleted(Event): + """An operation has completed""" + def __init__(self, total, msg = "Operation Completed"): + Event.__init__(self) + self.total = total + self.msg = msg + +class OperationProgress(Event): + """An operation is in progress""" + def __init__(self, current, total, msg = "Operation in Progress"): + Event.__init__(self) + self.current = current + self.total = total + self.msg = msg + ": %s/%s" % (current, total); + +class ConfigParsed(Event): + """Configuration Parsing Complete""" + +class RecipeEvent(Event): + def __init__(self, fn): + self.fn = fn + Event.__init__(self) + +class RecipePreFinalise(RecipeEvent): + """ Recipe Parsing Complete but not yet finialised""" + +class RecipeParsed(RecipeEvent): + """ Recipe Parsing Complete """ + +class StampUpdate(Event): + """Trigger for any adjustment of the stamp files to happen""" + + def __init__(self, targets, stampfns): + self._targets = targets + self._stampfns = stampfns + Event.__init__(self) + + def getStampPrefix(self): + return self._stampfns + + def getTargets(self): + return self._targets + + stampPrefix = property(getStampPrefix) + targets = property(getTargets) + +class BuildBase(Event): + """Base class for bbmake run events""" + + def __init__(self, n, p, failures = 0): + self._name = n + self._pkgs = p + Event.__init__(self) + self._failures = failures + + def getPkgs(self): + return self._pkgs + + def setPkgs(self, pkgs): + self._pkgs = pkgs + + def getName(self): + return self._name + + def setName(self, name): + self._name = name + + def getCfg(self): + return self.data + + def setCfg(self, cfg): + self.data = cfg + + def getFailures(self): + """ + Return the number of failed packages + """ + return self._failures + + pkgs = property(getPkgs, setPkgs, None, "pkgs property") + name = property(getName, setName, None, "name property") + cfg = property(getCfg, setCfg, None, "cfg property") + + + + + +class BuildStarted(BuildBase, OperationStarted): + """bbmake build run started""" + def __init__(self, n, p, failures = 0): + OperationStarted.__init__(self, "Building Started") + BuildBase.__init__(self, n, p, failures) + +class BuildCompleted(BuildBase, OperationCompleted): + """bbmake build run completed""" + def __init__(self, total, n, p, failures=0, interrupted=0): + if not failures: + OperationCompleted.__init__(self, total, "Building Succeeded") + else: + OperationCompleted.__init__(self, total, "Building Failed") + self._interrupted = interrupted + BuildBase.__init__(self, n, p, failures) + +class DiskFull(Event): + """Disk full case build aborted""" + def __init__(self, dev, type, freespace, mountpoint): + Event.__init__(self) + self._dev = dev + self._type = type + self._free = freespace + self._mountpoint = mountpoint + +class NoProvider(Event): + """No Provider for an Event""" + + def __init__(self, item, runtime=False, dependees=None, reasons=None, close_matches=None): + Event.__init__(self) + self._item = item + self._runtime = runtime + self._dependees = dependees + self._reasons = reasons + self._close_matches = close_matches + + def getItem(self): + return self._item + + def isRuntime(self): + return self._runtime + +class MultipleProviders(Event): + """Multiple Providers""" + + def __init__(self, item, candidates, runtime = False): + Event.__init__(self) + self._item = item + self._candidates = candidates + self._is_runtime = runtime + + def isRuntime(self): + """ + Is this a runtime issue? + """ + return self._is_runtime + + def getItem(self): + """ + The name for the to be build item + """ + return self._item + + def getCandidates(self): + """ + Get the possible Candidates for a PROVIDER. + """ + return self._candidates + +class ParseStarted(OperationStarted): + """Recipe parsing for the runqueue has begun""" + def __init__(self, total): + OperationStarted.__init__(self, "Recipe parsing Started") + self.total = total + +class ParseCompleted(OperationCompleted): + """Recipe parsing for the runqueue has completed""" + def __init__(self, cached, parsed, skipped, masked, virtuals, errors, total): + OperationCompleted.__init__(self, total, "Recipe parsing Completed") + self.cached = cached + self.parsed = parsed + self.skipped = skipped + self.virtuals = virtuals + self.masked = masked + self.errors = errors + self.sofar = cached + parsed + +class ParseProgress(OperationProgress): + """Recipe parsing progress""" + def __init__(self, current, total): + OperationProgress.__init__(self, current, total, "Recipe parsing") + + +class CacheLoadStarted(OperationStarted): + """Loading of the dependency cache has begun""" + def __init__(self, total): + OperationStarted.__init__(self, "Loading cache Started") + self.total = total + +class CacheLoadProgress(OperationProgress): + """Cache loading progress""" + def __init__(self, current, total): + OperationProgress.__init__(self, current, total, "Loading cache") + +class CacheLoadCompleted(OperationCompleted): + """Cache loading is complete""" + def __init__(self, total, num_entries): + OperationCompleted.__init__(self, total, "Loading cache Completed") + self.num_entries = num_entries + +class TreeDataPreparationStarted(OperationStarted): + """Tree data preparation started""" + def __init__(self): + OperationStarted.__init__(self, "Preparing tree data Started") + +class TreeDataPreparationProgress(OperationProgress): + """Tree data preparation is in progress""" + def __init__(self, current, total): + OperationProgress.__init__(self, current, total, "Preparing tree data") + +class TreeDataPreparationCompleted(OperationCompleted): + """Tree data preparation completed""" + def __init__(self, total): + OperationCompleted.__init__(self, total, "Preparing tree data Completed") + +class DepTreeGenerated(Event): + """ + Event when a dependency tree has been generated + """ + + def __init__(self, depgraph): + Event.__init__(self) + self._depgraph = depgraph + +class TargetsTreeGenerated(Event): + """ + Event when a set of buildable targets has been generated + """ + def __init__(self, model): + Event.__init__(self) + self._model = model + +class ReachableStamps(Event): + """ + An event listing all stamps reachable after parsing + which the metadata may use to clean up stale data + """ + + def __init__(self, stamps): + Event.__init__(self) + self.stamps = stamps + +class FilesMatchingFound(Event): + """ + Event when a list of files matching the supplied pattern has + been generated + """ + def __init__(self, pattern, matches): + Event.__init__(self) + self._pattern = pattern + self._matches = matches + +class CoreBaseFilesFound(Event): + """ + Event when a list of appropriate config files has been generated + """ + def __init__(self, paths): + Event.__init__(self) + self._paths = paths + +class ConfigFilesFound(Event): + """ + Event when a list of appropriate config files has been generated + """ + def __init__(self, variable, values): + Event.__init__(self) + self._variable = variable + self._values = values + +class ConfigFilePathFound(Event): + """ + Event when a path for a config file has been found + """ + def __init__(self, path): + Event.__init__(self) + self._path = path + +class MsgBase(Event): + """Base class for messages""" + + def __init__(self, msg): + self._message = msg + Event.__init__(self) + +class MsgDebug(MsgBase): + """Debug Message""" + +class MsgNote(MsgBase): + """Note Message""" + +class MsgWarn(MsgBase): + """Warning Message""" + +class MsgError(MsgBase): + """Error Message""" + +class MsgFatal(MsgBase): + """Fatal Message""" + +class MsgPlain(MsgBase): + """General output""" + +class LogExecTTY(Event): + """Send event containing program to spawn on tty of the logger""" + def __init__(self, msg, prog, sleep_delay, retries): + Event.__init__(self) + self.msg = msg + self.prog = prog + self.sleep_delay = sleep_delay + self.retries = retries + +class LogHandler(logging.Handler): + """Dispatch logging messages as bitbake events""" + + def emit(self, record): + if record.exc_info: + etype, value, tb = record.exc_info + if hasattr(tb, 'tb_next'): + tb = list(bb.exceptions.extract_traceback(tb, context=3)) + # Need to turn the value into something the logging system can pickle + value = str(value) + record.bb_exc_info = (etype, value, tb) + record.exc_info = None + fire(record, None) + + def filter(self, record): + record.taskpid = worker_pid + return True + +class RequestPackageInfo(Event): + """ + Event to request package information + """ + +class PackageInfo(Event): + """ + Package information for GUI + """ + def __init__(self, pkginfolist): + Event.__init__(self) + self._pkginfolist = pkginfolist + +class MetadataEvent(Event): + """ + Generic event that target for OE-Core classes + to report information during asynchrous execution + """ + def __init__(self, eventtype, eventdata): + Event.__init__(self) + self.type = eventtype + self._localdata = eventdata + +class SanityCheck(Event): + """ + Event to run sanity checks, either raise errors or generate events as return status. + """ + def __init__(self, generateevents = True): + Event.__init__(self) + self.generateevents = generateevents + +class SanityCheckPassed(Event): + """ + Event to indicate sanity check has passed + """ + +class SanityCheckFailed(Event): + """ + Event to indicate sanity check has failed + """ + def __init__(self, msg, network_error=False): + Event.__init__(self) + self._msg = msg + self._network_error = network_error + +class NetworkTest(Event): + """ + Event to run network connectivity tests, either raise errors or generate events as return status. + """ + def __init__(self, generateevents = True): + Event.__init__(self) + self.generateevents = generateevents + +class NetworkTestPassed(Event): + """ + Event to indicate network test has passed + """ + +class NetworkTestFailed(Event): + """ + Event to indicate network test has failed + """ + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/exceptions.py b/import-layers/yocto-poky/bitbake/lib/bb/exceptions.py new file mode 100644 index 000000000..f182c8fd6 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/exceptions.py @@ -0,0 +1,91 @@ +from __future__ import absolute_import +import inspect +import traceback +import bb.namedtuple_with_abc +from collections import namedtuple + + +class TracebackEntry(namedtuple.abc): + """Pickleable representation of a traceback entry""" + _fields = 'filename lineno function args code_context index' + _header = ' File "{0.filename}", line {0.lineno}, in {0.function}{0.args}' + + def format(self, formatter=None): + if not self.code_context: + return self._header.format(self) + '\n' + + formatted = [self._header.format(self) + ':\n'] + + for lineindex, line in enumerate(self.code_context): + if formatter: + line = formatter(line) + + if lineindex == self.index: + formatted.append(' >%s' % line) + else: + formatted.append(' %s' % line) + return formatted + + def __str__(self): + return ''.join(self.format()) + +def _get_frame_args(frame): + """Get the formatted arguments and class (if available) for a frame""" + arginfo = inspect.getargvalues(frame) + + try: + if not arginfo.args: + return '', None + # There have been reports from the field of python 2.6 which doesn't + # return a namedtuple here but simply a tuple so fallback gracefully if + # args isn't present. + except AttributeError: + return '', None + + firstarg = arginfo.args[0] + if firstarg == 'self': + self = arginfo.locals['self'] + cls = self.__class__.__name__ + + arginfo.args.pop(0) + del arginfo.locals['self'] + else: + cls = None + + formatted = inspect.formatargvalues(*arginfo) + return formatted, cls + +def extract_traceback(tb, context=1): + frames = inspect.getinnerframes(tb, context) + for frame, filename, lineno, function, code_context, index in frames: + formatted_args, cls = _get_frame_args(frame) + if cls: + function = '%s.%s' % (cls, function) + yield TracebackEntry(filename, lineno, function, formatted_args, + code_context, index) + +def format_extracted(extracted, formatter=None, limit=None): + if limit: + extracted = extracted[-limit:] + + formatted = [] + for tracebackinfo in extracted: + formatted.extend(tracebackinfo.format(formatter)) + return formatted + + +def format_exception(etype, value, tb, context=1, limit=None, formatter=None): + formatted = ['Traceback (most recent call last):\n'] + + if hasattr(tb, 'tb_next'): + tb = extract_traceback(tb, context) + + formatted.extend(format_extracted(tb, formatter, limit)) + formatted.extend(traceback.format_exception_only(etype, value)) + return formatted + +def to_string(exc): + if isinstance(exc, SystemExit): + if not isinstance(exc.code, basestring): + return 'Exited with "%d"' % exc.code + return str(exc) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py new file mode 100644 index 000000000..1fa67020c --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py @@ -0,0 +1,1773 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. +""" + +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2012 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +from __future__ import absolute_import +from __future__ import print_function +import os, re +import signal +import logging +import urllib +import urlparse +import bb.persist_data, bb.utils +import bb.checksum +from bb import data +import bb.process +import subprocess + +__version__ = "2" +_checksum_cache = bb.checksum.FileChecksumCache() + +logger = logging.getLogger("BitBake.Fetcher") + +try: + import cPickle as pickle +except ImportError: + import pickle + logger.info("Importing cPickle failed. " + "Falling back to a very slow implementation.") + +class BBFetchException(Exception): + """Class all fetch exceptions inherit from""" + def __init__(self, message): + self.msg = message + Exception.__init__(self, message) + + def __str__(self): + return self.msg + +class UntrustedUrl(BBFetchException): + """Exception raised when encountering a host not listed in BB_ALLOWED_NETWORKS""" + def __init__(self, url, message=''): + if message: + msg = message + else: + msg = "The URL: '%s' is not trusted and cannot be used" % url + self.url = url + BBFetchException.__init__(self, msg) + self.args = (url,) + +class MalformedUrl(BBFetchException): + """Exception raised when encountering an invalid url""" + def __init__(self, url, message=''): + if message: + msg = message + else: + msg = "The URL: '%s' is invalid and cannot be interpreted" % url + self.url = url + BBFetchException.__init__(self, msg) + self.args = (url,) + +class FetchError(BBFetchException): + """General fetcher exception when something happens incorrectly""" + def __init__(self, message, url = None): + if url: + msg = "Fetcher failure for URL: '%s'. %s" % (url, message) + else: + msg = "Fetcher failure: %s" % message + self.url = url + BBFetchException.__init__(self, msg) + self.args = (message, url) + +class ChecksumError(FetchError): + """Exception when mismatched checksum encountered""" + def __init__(self, message, url = None, checksum = None): + self.checksum = checksum + FetchError.__init__(self, message, url) + +class NoChecksumError(FetchError): + """Exception when no checksum is specified, but BB_STRICT_CHECKSUM is set""" + +class UnpackError(BBFetchException): + """General fetcher exception when something happens incorrectly when unpacking""" + def __init__(self, message, url): + msg = "Unpack failure for URL: '%s'. %s" % (url, message) + self.url = url + BBFetchException.__init__(self, msg) + self.args = (message, url) + +class NoMethodError(BBFetchException): + """Exception raised when there is no method to obtain a supplied url or set of urls""" + def __init__(self, url): + msg = "Could not find a fetcher which supports the URL: '%s'" % url + self.url = url + BBFetchException.__init__(self, msg) + self.args = (url,) + +class MissingParameterError(BBFetchException): + """Exception raised when a fetch method is missing a critical parameter in the url""" + def __init__(self, missing, url): + msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing) + self.url = url + self.missing = missing + BBFetchException.__init__(self, msg) + self.args = (missing, url) + +class ParameterError(BBFetchException): + """Exception raised when a url cannot be proccessed due to invalid parameters.""" + def __init__(self, message, url): + msg = "URL: '%s' has invalid parameters. %s" % (url, message) + self.url = url + BBFetchException.__init__(self, msg) + self.args = (message, url) + +class NetworkAccess(BBFetchException): + """Exception raised when network access is disabled but it is required.""" + def __init__(self, url, cmd): + msg = "Network access disabled through BB_NO_NETWORK (or set indirectly due to use of BB_FETCH_PREMIRRORONLY) but access requested with command %s (for url %s)" % (cmd, url) + self.url = url + self.cmd = cmd + BBFetchException.__init__(self, msg) + self.args = (url, cmd) + +class NonLocalMethod(Exception): + def __init__(self): + Exception.__init__(self) + + +class URI(object): + """ + A class representing a generic URI, with methods for + accessing the URI components, and stringifies to the + URI. + + It is constructed by calling it with a URI, or setting + the attributes manually: + + uri = URI("http://example.com/") + + uri = URI() + uri.scheme = 'http' + uri.hostname = 'example.com' + uri.path = '/' + + It has the following attributes: + + * scheme (read/write) + * userinfo (authentication information) (read/write) + * username (read/write) + * password (read/write) + + Note, password is deprecated as of RFC 3986. + + * hostname (read/write) + * port (read/write) + * hostport (read only) + "hostname:port", if both are set, otherwise just "hostname" + * path (read/write) + * path_quoted (read/write) + A URI quoted version of path + * params (dict) (read/write) + * query (dict) (read/write) + * relative (bool) (read only) + True if this is a "relative URI", (e.g. file:foo.diff) + + It stringifies to the URI itself. + + Some notes about relative URIs: while it's specified that + a URI beginning with <scheme>:// should either be directly + followed by a hostname or a /, the old URI handling of the + fetch2 library did not comform to this. Therefore, this URI + class has some kludges to make sure that URIs are parsed in + a way comforming to bitbake's current usage. This URI class + supports the following: + + file:relative/path.diff (IETF compliant) + git:relative/path.git (IETF compliant) + git:///absolute/path.git (IETF compliant) + file:///absolute/path.diff (IETF compliant) + + file://relative/path.diff (not IETF compliant) + + But it does not support the following: + + file://hostname/absolute/path.diff (would be IETF compliant) + + Note that the last case only applies to a list of + "whitelisted" schemes (currently only file://), that requires + its URIs to not have a network location. + """ + + _relative_schemes = ['file', 'git'] + _netloc_forbidden = ['file'] + + def __init__(self, uri=None): + self.scheme = '' + self.userinfo = '' + self.hostname = '' + self.port = None + self._path = '' + self.params = {} + self.query = {} + self.relative = False + + if not uri: + return + + # We hijack the URL parameters, since the way bitbake uses + # them are not quite RFC compliant. + uri, param_str = (uri.split(";", 1) + [None])[:2] + + urlp = urlparse.urlparse(uri) + self.scheme = urlp.scheme + + reparse = 0 + + # Coerce urlparse to make URI scheme use netloc + if not self.scheme in urlparse.uses_netloc: + urlparse.uses_params.append(self.scheme) + reparse = 1 + + # Make urlparse happy(/ier) by converting local resources + # to RFC compliant URL format. E.g.: + # file://foo.diff -> file:foo.diff + if urlp.scheme in self._netloc_forbidden: + uri = re.sub("(?<=:)//(?!/)", "", uri, 1) + reparse = 1 + + if reparse: + urlp = urlparse.urlparse(uri) + + # Identify if the URI is relative or not + if urlp.scheme in self._relative_schemes and \ + re.compile("^\w+:(?!//)").match(uri): + self.relative = True + + if not self.relative: + self.hostname = urlp.hostname or '' + self.port = urlp.port + + self.userinfo += urlp.username or '' + + if urlp.password: + self.userinfo += ':%s' % urlp.password + + self.path = urllib.unquote(urlp.path) + + if param_str: + self.params = self._param_str_split(param_str, ";") + if urlp.query: + self.query = self._param_str_split(urlp.query, "&") + + def __str__(self): + userinfo = self.userinfo + if userinfo: + userinfo += '@' + + return "%s:%s%s%s%s%s%s" % ( + self.scheme, + '' if self.relative else '//', + userinfo, + self.hostport, + self.path_quoted, + self._query_str(), + self._param_str()) + + def _param_str(self): + return ( + ''.join([';', self._param_str_join(self.params, ";")]) + if self.params else '') + + def _query_str(self): + return ( + ''.join(['?', self._param_str_join(self.query, "&")]) + if self.query else '') + + def _param_str_split(self, string, elmdelim, kvdelim="="): + ret = {} + for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim)]: + ret[k] = v + return ret + + def _param_str_join(self, dict_, elmdelim, kvdelim="="): + return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()]) + + @property + def hostport(self): + if not self.port: + return self.hostname + return "%s:%d" % (self.hostname, self.port) + + @property + def path_quoted(self): + return urllib.quote(self.path) + + @path_quoted.setter + def path_quoted(self, path): + self.path = urllib.unquote(path) + + @property + def path(self): + return self._path + + @path.setter + def path(self, path): + self._path = path + + if not path or re.compile("^/").match(path): + self.relative = False + else: + self.relative = True + + @property + def username(self): + if self.userinfo: + return (self.userinfo.split(":", 1))[0] + return '' + + @username.setter + def username(self, username): + password = self.password + self.userinfo = username + if password: + self.userinfo += ":%s" % password + + @property + def password(self): + if self.userinfo and ":" in self.userinfo: + return (self.userinfo.split(":", 1))[1] + return '' + + @password.setter + def password(self, password): + self.userinfo = "%s:%s" % (self.username, password) + +def decodeurl(url): + """Decodes an URL into the tokens (scheme, network location, path, + user, password, parameters). + """ + + m = re.compile('(?P<type>[^:]*)://((?P<user>[^/]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url) + if not m: + raise MalformedUrl(url) + + type = m.group('type') + location = m.group('location') + if not location: + raise MalformedUrl(url) + user = m.group('user') + parm = m.group('parm') + + locidx = location.find('/') + if locidx != -1 and type.lower() != 'file': + host = location[:locidx] + path = location[locidx:] + elif type.lower() == 'file': + host = "" + path = location + else: + host = location + path = "" + if user: + m = re.compile('(?P<user>[^:]+)(:?(?P<pswd>.*))').match(user) + if m: + user = m.group('user') + pswd = m.group('pswd') + else: + user = '' + pswd = '' + + p = {} + if parm: + for s in parm.split(';'): + if s: + if not '=' in s: + raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s)) + s1, s2 = s.split('=') + p[s1] = s2 + + return type, host, urllib.unquote(path), user, pswd, p + +def encodeurl(decoded): + """Encodes a URL from tokens (scheme, network location, path, + user, password, parameters). + """ + + type, host, path, user, pswd, p = decoded + + if not path: + raise MissingParameterError('path', "encoded from the data %s" % str(decoded)) + if not type: + raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) + url = '%s://' % type + if user and type != "file": + url += "%s" % user + if pswd: + url += ":%s" % pswd + url += "@" + if host and type != "file": + url += "%s" % host + # Standardise path to ensure comparisons work + while '//' in path: + path = path.replace("//", "/") + url += "%s" % urllib.quote(path) + if p: + for parm in p: + url += ";%s=%s" % (parm, p[parm]) + + return url + +def uri_replace(ud, uri_find, uri_replace, replacements, d): + if not ud.url or not uri_find or not uri_replace: + logger.error("uri_replace: passed an undefined value, not replacing") + return None + uri_decoded = list(decodeurl(ud.url)) + uri_find_decoded = list(decodeurl(uri_find)) + uri_replace_decoded = list(decodeurl(uri_replace)) + logger.debug(2, "For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded)) + result_decoded = ['', '', '', '', '', {}] + for loc, i in enumerate(uri_find_decoded): + result_decoded[loc] = uri_decoded[loc] + regexp = i + if loc == 0 and regexp and not regexp.endswith("$"): + # Leaving the type unanchored can mean "https" matching "file" can become "files" + # which is clearly undesirable. + regexp += "$" + if loc == 5: + # Handle URL parameters + if i: + # Any specified URL parameters must match + for k in uri_replace_decoded[loc]: + if uri_decoded[loc][k] != uri_replace_decoded[loc][k]: + return None + # Overwrite any specified replacement parameters + for k in uri_replace_decoded[loc]: + for l in replacements: + uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l]) + result_decoded[loc][k] = uri_replace_decoded[loc][k] + elif (re.match(regexp, uri_decoded[loc])): + if not uri_replace_decoded[loc]: + result_decoded[loc] = "" + else: + for k in replacements: + uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k]) + #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc])) + result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1) + if loc == 2: + # Handle path manipulations + basename = None + if uri_decoded[0] != uri_replace_decoded[0] and ud.mirrortarball: + # If the source and destination url types differ, must be a mirrortarball mapping + basename = os.path.basename(ud.mirrortarball) + # Kill parameters, they make no sense for mirror tarballs + uri_decoded[5] = {} + elif ud.localpath and ud.method.supports_checksum(ud): + basename = os.path.basename(ud.localpath) + if basename and not result_decoded[loc].endswith(basename): + result_decoded[loc] = os.path.join(result_decoded[loc], basename) + else: + return None + result = encodeurl(result_decoded) + if result == ud.url: + return None + logger.debug(2, "For url %s returning %s" % (ud.url, result)) + return result + +methods = [] +urldata_cache = {} +saved_headrevs = {} + +def fetcher_init(d): + """ + Called to initialize the fetchers once the configuration data is known. + Calls before this must not hit the cache. + """ + # When to drop SCM head revisions controlled by user policy + srcrev_policy = d.getVar('BB_SRCREV_POLICY', True) or "clear" + if srcrev_policy == "cache": + logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy) + elif srcrev_policy == "clear": + logger.debug(1, "Clearing SRCREV cache due to cache policy of: %s", srcrev_policy) + revs = bb.persist_data.persist('BB_URI_HEADREVS', d) + try: + bb.fetch2.saved_headrevs = revs.items() + except: + pass + revs.clear() + else: + raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) + + _checksum_cache.init_cache(d) + + for m in methods: + if hasattr(m, "init"): + m.init(d) + +def fetcher_parse_save(): + _checksum_cache.save_extras() + +def fetcher_parse_done(): + _checksum_cache.save_merge() + +def fetcher_compare_revisions(): + """ + Compare the revisions in the persistant cache with current values and + return true/false on whether they've changed. + """ + + data = bb.persist_data.persist('BB_URI_HEADREVS', d).items() + data2 = bb.fetch2.saved_headrevs + + changed = False + for key in data: + if key not in data2 or data2[key] != data[key]: + logger.debug(1, "%s changed", key) + changed = True + return True + else: + logger.debug(2, "%s did not change", key) + return False + +def mirror_from_string(data): + return [ i.split() for i in (data or "").replace('\\n','\n').split('\n') if i ] + +def verify_checksum(ud, d, precomputed={}): + """ + verify the MD5 and SHA256 checksum for downloaded src + + Raises a FetchError if one or both of the SRC_URI checksums do not match + the downloaded file, or if BB_STRICT_CHECKSUM is set and there are no + checksums specified. + + Returns a dict of checksums that can be stored in a done stamp file and + passed in as precomputed parameter in a later call to avoid re-computing + the checksums from the file. This allows verifying the checksums of the + file against those in the recipe each time, rather than only after + downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. + """ + + _MD5_KEY = "md5" + _SHA256_KEY = "sha256" + + if ud.ignore_checksums or not ud.method.supports_checksum(ud): + return {} + + if _MD5_KEY in precomputed: + md5data = precomputed[_MD5_KEY] + else: + md5data = bb.utils.md5_file(ud.localpath) + + if _SHA256_KEY in precomputed: + sha256data = precomputed[_SHA256_KEY] + else: + sha256data = bb.utils.sha256_file(ud.localpath) + + if ud.method.recommends_checksum(ud) and not ud.md5_expected and not ud.sha256_expected: + # If strict checking enabled and neither sum defined, raise error + strict = d.getVar("BB_STRICT_CHECKSUM", True) or "0" + if strict == "1": + logger.error('No checksum specified for %s, please add at least one to the recipe:\n' + 'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' % + (ud.localpath, ud.md5_name, md5data, + ud.sha256_name, sha256data)) + raise NoChecksumError('Missing SRC_URI checksum', ud.url) + + # Log missing sums so user can more easily add them + logger.warn('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n' + 'SRC_URI[%s] = "%s"', + ud.localpath, ud.md5_name, md5data) + logger.warn('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n' + 'SRC_URI[%s] = "%s"', + ud.localpath, ud.sha256_name, sha256data) + + # We want to alert the user if a checksum is defined in the recipe but + # it does not match. + msg = "" + mismatch = False + if ud.md5_expected and ud.md5_expected != md5data: + msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'md5', md5data, ud.md5_expected) + mismatch = True; + + if ud.sha256_expected and ud.sha256_expected != sha256data: + msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'sha256', sha256data, ud.sha256_expected) + mismatch = True; + + if mismatch: + msg = msg + '\nIf this change is expected (e.g. you have upgraded to a new version without updating the checksums) then you can use these lines within the recipe:\nSRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"\nOtherwise you should retry the download and/or check with upstream to determine if the file has become corrupted or otherwise unexpectedly modified.\n' % (ud.md5_name, md5data, ud.sha256_name, sha256data) + + if len(msg): + raise ChecksumError('Checksum mismatch!%s' % msg, ud.url, md5data) + + return { + _MD5_KEY: md5data, + _SHA256_KEY: sha256data + } + + +def verify_donestamp(ud, d, origud=None): + """ + Check whether the done stamp file has the right checksums (if the fetch + method supports them). If it doesn't, delete the done stamp and force + a re-download. + + Returns True, if the donestamp exists and is valid, False otherwise. When + returning False, any existing done stamps are removed. + """ + if not ud.needdonestamp: + return True + + if not os.path.exists(ud.donestamp): + return False + + if (not ud.method.supports_checksum(ud) or + (origud and not origud.method.supports_checksum(origud))): + # done stamp exists, checksums not supported; assume the local file is + # current + return True + + if not os.path.exists(ud.localpath): + # done stamp exists, but the downloaded file does not; the done stamp + # must be incorrect, re-trigger the download + bb.utils.remove(ud.donestamp) + return False + + precomputed_checksums = {} + # Only re-use the precomputed checksums if the donestamp is newer than the + # file. Do not rely on the mtime of directories, though. If ud.localpath is + # a directory, there will probably not be any checksums anyway. + if (os.path.isdir(ud.localpath) or + os.path.getmtime(ud.localpath) < os.path.getmtime(ud.donestamp)): + try: + with open(ud.donestamp, "rb") as cachefile: + pickled = pickle.Unpickler(cachefile) + precomputed_checksums.update(pickled.load()) + except Exception as e: + # Avoid the warnings on the upgrade path from emtpy done stamp + # files to those containing the checksums. + if not isinstance(e, EOFError): + # Ignore errors, they aren't fatal + logger.warn("Couldn't load checksums from donestamp %s: %s " + "(msg: %s)" % (ud.donestamp, type(e).__name__, + str(e))) + + try: + checksums = verify_checksum(ud, d, precomputed_checksums) + # If the cache file did not have the checksums, compute and store them + # as an upgrade path from the previous done stamp file format. + if checksums != precomputed_checksums: + with open(ud.donestamp, "wb") as cachefile: + p = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL) + p.dump(checksums) + return True + except ChecksumError as e: + # Checksums failed to verify, trigger re-download and remove the + # incorrect stamp file. + logger.warn("Checksum mismatch for local file %s\n" + "Cleaning and trying again." % ud.localpath) + if os.path.exists(ud.localpath): + rename_bad_checksum(ud, e.checksum) + bb.utils.remove(ud.donestamp) + return False + + +def update_stamp(ud, d): + """ + donestamp is file stamp indicating the whole fetching is done + this function update the stamp after verifying the checksum + """ + if not ud.needdonestamp: + return + + if os.path.exists(ud.donestamp): + # Touch the done stamp file to show active use of the download + try: + os.utime(ud.donestamp, None) + except: + # Errors aren't fatal here + pass + else: + try: + checksums = verify_checksum(ud, d) + # Store the checksums for later re-verification against the recipe + with open(ud.donestamp, "wb") as cachefile: + p = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL) + p.dump(checksums) + except ChecksumError as e: + # Checksums failed to verify, trigger re-download and remove the + # incorrect stamp file. + logger.warn("Checksum mismatch for local file %s\n" + "Cleaning and trying again." % ud.localpath) + if os.path.exists(ud.localpath): + rename_bad_checksum(ud, e.checksum) + bb.utils.remove(ud.donestamp) + raise + +def subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + # SIGPIPE errors are known issues with gzip/bash + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +def get_autorev(d): + # only not cache src rev in autorev case + if d.getVar('BB_SRCREV_POLICY', True) != "cache": + d.setVar('BB_DONT_CACHE', '1') + return "AUTOINC" + +def get_srcrev(d, method_name='sortable_revision'): + """ + Return the revsion string, usually for use in the version string (PV) of the current package + Most packages usually only have one SCM so we just pass on the call. + In the multi SCM case, we build a value based on SRCREV_FORMAT which must + have been set. + + The idea here is that we put the string "AUTOINC+" into return value if the revisions are not + incremental, other code is then responsible for turning that into an increasing value (if needed) + + A method_name can be supplied to retrieve an alternatively formatted revision from a fetcher, if + that fetcher provides a method with the given name and the same signature as sortable_revision. + """ + + scms = [] + fetcher = Fetch(d.getVar('SRC_URI', True).split(), d) + urldata = fetcher.ud + for u in urldata: + if urldata[u].method.supports_srcrev(): + scms.append(u) + + if len(scms) == 0: + raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") + + if len(scms) == 1 and len(urldata[scms[0]].names) == 1: + autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0]) + if len(rev) > 10: + rev = rev[:10] + if autoinc: + return "AUTOINC+" + rev + return rev + + # + # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT + # + format = d.getVar('SRCREV_FORMAT', True) + if not format: + raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.") + + seenautoinc = False + for scm in scms: + ud = urldata[scm] + for name in ud.names: + autoinc, rev = getattr(ud.method, method_name)(ud, d, name) + seenautoinc = seenautoinc or autoinc + if len(rev) > 10: + rev = rev[:10] + format = format.replace(name, rev) + if seenautoinc: + format = "AUTOINC+" + format + + return format + +def localpath(url, d): + fetcher = bb.fetch2.Fetch([url], d) + return fetcher.localpath(url) + +def runfetchcmd(cmd, d, quiet=False, cleanup=None): + """ + Run cmd returning the command output + Raise an error if interrupted or cmd fails + Optionally echo command output to stdout + Optionally remove the files/directories listed in cleanup upon failure + """ + + # Need to export PATH as binary could be in metadata paths + # rather than host provided + # Also include some other variables. + # FIXME: Should really include all export varaiables? + exportvars = ['HOME', 'PATH', + 'HTTP_PROXY', 'http_proxy', + 'HTTPS_PROXY', 'https_proxy', + 'FTP_PROXY', 'ftp_proxy', + 'FTPS_PROXY', 'ftps_proxy', + 'NO_PROXY', 'no_proxy', + 'ALL_PROXY', 'all_proxy', + 'GIT_PROXY_COMMAND', + 'GIT_SSL_CAINFO', + 'GIT_SMART_HTTP', + 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', + 'SOCKS5_USER', 'SOCKS5_PASSWD'] + + if not cleanup: + cleanup = [] + + for var in exportvars: + val = d.getVar(var, True) + if val: + cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) + + logger.debug(1, "Running %s", cmd) + + success = False + error_message = "" + + try: + (output, errors) = bb.process.run(cmd, shell=True, stderr=subprocess.PIPE) + success = True + except bb.process.NotFoundError as e: + error_message = "Fetch command %s" % (e.command) + except bb.process.ExecutionError as e: + if e.stdout: + output = "output:\n%s\n%s" % (e.stdout, e.stderr) + elif e.stderr: + output = "output:\n%s" % e.stderr + else: + output = "no output" + error_message = "Fetch command failed with exit code %s, %s" % (e.exitcode, output) + except bb.process.CmdError as e: + error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg) + if not success: + for f in cleanup: + try: + bb.utils.remove(f, True) + except OSError: + pass + + raise FetchError(error_message) + + return output + +def check_network_access(d, info = "", url = None): + """ + log remote network access, and error if BB_NO_NETWORK is set + """ + if d.getVar("BB_NO_NETWORK", True) == "1": + raise NetworkAccess(url, info) + else: + logger.debug(1, "Fetcher accessed the network with the command %s" % info) + +def build_mirroruris(origud, mirrors, ld): + uris = [] + uds = [] + + replacements = {} + replacements["TYPE"] = origud.type + replacements["HOST"] = origud.host + replacements["PATH"] = origud.path + replacements["BASENAME"] = origud.path.split("/")[-1] + replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') + + def adduri(ud, uris, uds, mirrors): + for line in mirrors: + try: + (find, replace) = line + except ValueError: + continue + newuri = uri_replace(ud, find, replace, replacements, ld) + if not newuri or newuri in uris or newuri == origud.url: + continue + + if not trusted_network(ld, newuri): + logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri)) + continue + + # Create a local copy of the mirrors minus the current line + # this will prevent us from recursively processing the same line + # as well as indirect recursion A -> B -> C -> A + localmirrors = list(mirrors) + localmirrors.remove(line) + + try: + newud = FetchData(newuri, ld) + newud.setup_localpath(ld) + except bb.fetch2.BBFetchException as e: + logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) + logger.debug(1, str(e)) + try: + # setup_localpath of file:// urls may fail, we should still see + # if mirrors of the url exist + adduri(newud, uris, uds, localmirrors) + except UnboundLocalError: + pass + continue + uris.append(newuri) + uds.append(newud) + + adduri(newud, uris, uds, localmirrors) + + adduri(origud, uris, uds, mirrors) + + return uris, uds + +def rename_bad_checksum(ud, suffix): + """ + Renames files to have suffix from parameter + """ + + if ud.localpath is None: + return + + new_localpath = "%s_bad-checksum_%s" % (ud.localpath, suffix) + bb.warn("Renaming %s to %s" % (ud.localpath, new_localpath)) + bb.utils.movefile(ud.localpath, new_localpath) + + +def try_mirror_url(fetch, origud, ud, ld, check = False): + # Return of None or a value means we're finished + # False means try another url + + if ud.lockfile and ud.lockfile != origud.lockfile: + lf = bb.utils.lockfile(ud.lockfile) + + try: + if check: + found = ud.method.checkstatus(fetch, ud, ld) + if found: + return found + return False + + os.chdir(ld.getVar("DL_DIR", True)) + + if not verify_donestamp(ud, ld, origud) or ud.method.need_update(ud, ld): + ud.method.download(ud, ld) + if hasattr(ud.method,"build_mirror_data"): + ud.method.build_mirror_data(ud, ld) + + if not ud.localpath or not os.path.exists(ud.localpath): + return False + + if ud.localpath == origud.localpath: + return ud.localpath + + # We may be obtaining a mirror tarball which needs further processing by the real fetcher + # If that tarball is a local file:// we need to provide a symlink to it + dldir = ld.getVar("DL_DIR", True) + if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \ + and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): + # Create donestamp in old format to avoid triggering a re-download + if ud.donestamp: + bb.utils.mkdirhier(os.path.dirname(ud.donestamp)) + open(ud.donestamp, 'w').close() + dest = os.path.join(dldir, os.path.basename(ud.localpath)) + if not os.path.exists(dest): + os.symlink(ud.localpath, dest) + if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld): + origud.method.download(origud, ld) + if hasattr(origud.method,"build_mirror_data"): + origud.method.build_mirror_data(origud, ld) + return origud.localpath + # Otherwise the result is a local file:// and we symlink to it + if not os.path.exists(origud.localpath): + if os.path.islink(origud.localpath): + # Broken symbolic link + os.unlink(origud.localpath) + + os.symlink(ud.localpath, origud.localpath) + update_stamp(origud, ld) + return ud.localpath + + except bb.fetch2.NetworkAccess: + raise + + except bb.fetch2.BBFetchException as e: + if isinstance(e, ChecksumError): + logger.warn("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (ud.url, origud.url)) + logger.warn(str(e)) + if os.path.exists(ud.localpath): + rename_bad_checksum(ud, e.checksum) + elif isinstance(e, NoChecksumError): + raise + else: + logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url)) + logger.debug(1, str(e)) + try: + ud.method.clean(ud, ld) + except UnboundLocalError: + pass + return False + finally: + if ud.lockfile and ud.lockfile != origud.lockfile: + bb.utils.unlockfile(lf) + + +def try_mirrors(fetch, d, origud, mirrors, check = False): + """ + Try to use a mirrored version of the sources. + This method will be automatically called before the fetchers go. + + d Is a bb.data instance + uri is the original uri we're trying to download + mirrors is the list of mirrors we're going to try + """ + ld = d.createCopy() + + uris, uds = build_mirroruris(origud, mirrors, ld) + + for index, uri in enumerate(uris): + ret = try_mirror_url(fetch, origud, uds[index], ld, check) + if ret != False: + return ret + return None + +def trusted_network(d, url): + """ + Use a trusted url during download if networking is enabled and + BB_ALLOWED_NETWORKS is set globally or for a specific recipe. + Note: modifies SRC_URI & mirrors. + """ + if d.getVar('BB_NO_NETWORK', True) == "1": + return True + + pkgname = d.expand(d.getVar('PN', False)) + trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False) + + if not trusted_hosts: + trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS', True) + + # Not enabled. + if not trusted_hosts: + return True + + scheme, network, path, user, passwd, param = decodeurl(url) + + if not network: + return True + + network = network.split(':')[0] + network = network.lower() + + for host in trusted_hosts.split(" "): + host = host.lower() + if host.startswith("*.") and ("." + network).endswith(host[1:]): + return True + if host == network: + return True + + return False + +def srcrev_internal_helper(ud, d, name): + """ + Return: + a) a source revision if specified + b) latest revision if SRCREV="AUTOINC" + c) None if not specified + """ + + srcrev = None + pn = d.getVar("PN", True) + attempts = [] + if name != '' and pn: + attempts.append("SRCREV_%s_pn-%s" % (name, pn)) + if name != '': + attempts.append("SRCREV_%s" % name) + if pn: + attempts.append("SRCREV_pn-%s" % pn) + attempts.append("SRCREV") + + for a in attempts: + srcrev = d.getVar(a, True) + if srcrev and srcrev != "INVALID": + break + + if 'rev' in ud.parm and 'tag' in ud.parm: + raise FetchError("Please specify a ;rev= parameter or a ;tag= parameter in the url %s but not both." % (ud.url)) + + if 'rev' in ud.parm or 'tag' in ud.parm: + if 'rev' in ud.parm: + parmrev = ud.parm['rev'] + else: + parmrev = ud.parm['tag'] + if srcrev == "INVALID" or not srcrev: + return parmrev + if srcrev != parmrev: + raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please spcify one valid value" % (srcrev, parmrev)) + return parmrev + + if srcrev == "INVALID" or not srcrev: + raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url) + if srcrev == "AUTOINC": + srcrev = ud.method.latest_revision(ud, d, name) + + return srcrev + +def get_checksum_file_list(d): + """ Get a list of files checksum in SRC_URI + + Returns the resolved local paths of all local file entries in + SRC_URI as a space-separated string + """ + fetch = Fetch([], d, cache = False, localonly = True) + + dl_dir = d.getVar('DL_DIR', True) + filelist = [] + for u in fetch.urls: + ud = fetch.ud[u] + + if ud and isinstance(ud.method, local.Local): + paths = ud.method.localpaths(ud, d) + for f in paths: + pth = ud.decodedurl + if '*' in pth: + f = os.path.join(os.path.abspath(f), pth) + if f.startswith(dl_dir): + # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else + if os.path.exists(f): + bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN', True), os.path.basename(f))) + else: + bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN', True), os.path.basename(f))) + filelist.append(f + ":" + str(os.path.exists(f))) + + return " ".join(filelist) + +def get_file_checksums(filelist, pn): + """Get a list of the checksums for a list of local files + + Returns the checksums for a list of local files, caching the results as + it proceeds + + """ + return _checksum_cache.get_checksums(filelist, pn) + + +class FetchData(object): + """ + A class which represents the fetcher state for a given URI. + """ + def __init__(self, url, d, localonly = False): + # localpath is the location of a downloaded result. If not set, the file is local. + self.donestamp = None + self.needdonestamp = True + self.localfile = "" + self.localpath = None + self.lockfile = None + self.mirrortarball = None + self.basename = None + self.basepath = None + (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(data.expand(url, d)) + self.date = self.getSRCDate(d) + self.url = url + if not self.user and "user" in self.parm: + self.user = self.parm["user"] + if not self.pswd and "pswd" in self.parm: + self.pswd = self.parm["pswd"] + self.setup = False + + if "name" in self.parm: + self.md5_name = "%s.md5sum" % self.parm["name"] + self.sha256_name = "%s.sha256sum" % self.parm["name"] + else: + self.md5_name = "md5sum" + self.sha256_name = "sha256sum" + if self.md5_name in self.parm: + self.md5_expected = self.parm[self.md5_name] + elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]: + self.md5_expected = None + else: + self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name, True) + if self.sha256_name in self.parm: + self.sha256_expected = self.parm[self.sha256_name] + elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]: + self.sha256_expected = None + else: + self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name, True) + self.ignore_checksums = False + + self.names = self.parm.get("name",'default').split(',') + + self.method = None + for m in methods: + if m.supports(self, d): + self.method = m + break + + if not self.method: + raise NoMethodError(url) + + if localonly and not isinstance(self.method, local.Local): + raise NonLocalMethod() + + if self.parm.get("proto", None) and "protocol" not in self.parm: + logger.warn('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN', True)) + self.parm["protocol"] = self.parm.get("proto", None) + + if hasattr(self.method, "urldata_init"): + self.method.urldata_init(self, d) + + if "localpath" in self.parm: + # if user sets localpath for file, use it instead. + self.localpath = self.parm["localpath"] + self.basename = os.path.basename(self.localpath) + elif self.localfile: + self.localpath = self.method.localpath(self, d) + + dldir = d.getVar("DL_DIR", True) + + if not self.needdonestamp: + return + + # Note: .done and .lock files should always be in DL_DIR whereas localpath may not be. + if self.localpath and self.localpath.startswith(dldir): + basepath = self.localpath + elif self.localpath: + basepath = dldir + os.sep + os.path.basename(self.localpath) + elif self.basepath or self.basename: + basepath = dldir + os.sep + (self.basepath or self.basename) + else: + bb.fatal("Can't determine lock path for url %s" % url) + + self.donestamp = basepath + '.done' + self.lockfile = basepath + '.lock' + + def setup_revisons(self, d): + self.revisions = {} + for name in self.names: + self.revisions[name] = srcrev_internal_helper(self, d, name) + + # add compatibility code for non name specified case + if len(self.names) == 1: + self.revision = self.revisions[self.names[0]] + + def setup_localpath(self, d): + if not self.localpath: + self.localpath = self.method.localpath(self, d) + + def getSRCDate(self, d): + """ + Return the SRC Date for the component + + d the bb.data module + """ + if "srcdate" in self.parm: + return self.parm['srcdate'] + + pn = d.getVar("PN", True) + + if pn: + return d.getVar("SRCDATE_%s" % pn, True) or d.getVar("SRCDATE", True) or d.getVar("DATE", True) + + return d.getVar("SRCDATE", True) or d.getVar("DATE", True) + +class FetchMethod(object): + """Base class for 'fetch'ing data""" + + def __init__(self, urls=None): + self.urls = [] + + def supports(self, urldata, d): + """ + Check to see if this fetch class supports a given url. + """ + return 0 + + def localpath(self, urldata, d): + """ + Return the local filename of a given url assuming a successful fetch. + Can also setup variables in urldata for use in go (saving code duplication + and duplicate code execution) + """ + return os.path.join(data.getVar("DL_DIR", d, True), urldata.localfile) + + def supports_checksum(self, urldata): + """ + Is localpath something that can be represented by a checksum? + """ + + # We cannot compute checksums for directories + if os.path.isdir(urldata.localpath) == True: + return False + if urldata.localpath.find("*") != -1: + return False + + return True + + def recommends_checksum(self, urldata): + """ + Is the backend on where checksumming is recommended (should warnings + be displayed if there is no checksum)? + """ + return False + + def _strip_leading_slashes(self, relpath): + """ + Remove leading slash as os.path.join can't cope + """ + while os.path.isabs(relpath): + relpath = relpath[1:] + return relpath + + def setUrls(self, urls): + self.__urls = urls + + def getUrls(self): + return self.__urls + + urls = property(getUrls, setUrls, None, "Urls property") + + def need_update(self, ud, d): + """ + Force a fetch, even if localpath exists? + """ + if os.path.exists(ud.localpath): + return False + return True + + def supports_srcrev(self): + """ + The fetcher supports auto source revisions (SRCREV) + """ + return False + + def download(self, urldata, d): + """ + Fetch urls + Assumes localpath was called first + """ + raise NoMethodError(url) + + def unpack(self, urldata, rootdir, data): + iterate = False + file = urldata.localpath + + # Localpath can't deal with 'dir/*' entries, so it converts them to '.', + # but it must be corrected back for local files copying + if urldata.basename == '*' and file.endswith('/.'): + file = '%s/%s' % (file.rstrip('/.'), urldata.path) + + try: + unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True) + except ValueError as exc: + bb.fatal("Invalid value for 'unpack' parameter for %s: %s" % + (file, urldata.parm.get('unpack'))) + + base, ext = os.path.splitext(file) + if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz']: + efile = os.path.join(rootdir, os.path.basename(base)) + else: + efile = file + cmd = None + + if unpack: + if file.endswith('.tar'): + cmd = 'tar x --no-same-owner -f %s' % file + elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): + cmd = 'tar xz --no-same-owner -f %s' % file + elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): + cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file + elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): + cmd = 'gzip -dc %s > %s' % (file, efile) + elif file.endswith('.bz2'): + cmd = 'bzip2 -dc %s > %s' % (file, efile) + elif file.endswith('.tar.xz'): + cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file + elif file.endswith('.xz'): + cmd = 'xz -dc %s > %s' % (file, efile) + elif file.endswith('.tar.lz'): + cmd = 'lzip -dc %s | tar x --no-same-owner -f -' % file + elif file.endswith('.lz'): + cmd = 'lzip -dc %s > %s' % (file, efile) + elif file.endswith('.zip') or file.endswith('.jar'): + try: + dos = bb.utils.to_boolean(urldata.parm.get('dos'), False) + except ValueError as exc: + bb.fatal("Invalid value for 'dos' parameter for %s: %s" % + (file, urldata.parm.get('dos'))) + cmd = 'unzip -q -o' + if dos: + cmd = '%s -a' % cmd + cmd = "%s '%s'" % (cmd, file) + elif file.endswith('.rpm') or file.endswith('.srpm'): + if 'extract' in urldata.parm: + unpack_file = urldata.parm.get('extract') + cmd = 'rpm2cpio.sh %s | cpio -id %s' % (file, unpack_file) + iterate = True + iterate_file = unpack_file + else: + cmd = 'rpm2cpio.sh %s | cpio -id' % (file) + elif file.endswith('.deb') or file.endswith('.ipk'): + cmd = 'ar -p %s data.tar.gz | zcat | tar --no-same-owner -xpf -' % file + elif file.endswith('.tar.7z'): + cmd = '7z x -so %s | tar xf - ' % file + elif file.endswith('.7z'): + cmd = '7za x -y %s 1>/dev/null' % file + + # If 'subdir' param exists, create a dir and use it as destination for unpack cmd + if 'subdir' in urldata.parm: + unpackdir = '%s/%s' % (rootdir, urldata.parm.get('subdir')) + bb.utils.mkdirhier(unpackdir) + else: + unpackdir = rootdir + + if not unpack or not cmd: + # If file == dest, then avoid any copies, as we already put the file into dest! + dest = os.path.join(unpackdir, os.path.basename(file)) + if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): + destdir = '.' + # For file:// entries all intermediate dirs in path must be created at destination + if urldata.type == "file": + # Trailing '/' does a copying to wrong place + urlpath = urldata.path.rstrip('/') + # Want files places relative to cwd so no leading '/' + urlpath = urlpath.lstrip('/') + if urlpath.find("/") != -1: + destdir = urlpath.rsplit("/", 1)[0] + '/' + bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) + cmd = 'cp -fpPR %s %s' % (file, destdir) + + if not cmd: + return + + # Change to unpackdir before executing command + save_cwd = os.getcwd(); + os.chdir(unpackdir) + + path = data.getVar('PATH', True) + if path: + cmd = "PATH=\"%s\" %s" % (path, cmd) + bb.note("Unpacking %s to %s/" % (file, os.getcwd())) + ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) + + os.chdir(save_cwd) + + if ret != 0: + raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), urldata.url) + + if iterate is True: + iterate_urldata = urldata + iterate_urldata.localpath = "%s/%s" % (rootdir, iterate_file) + self.unpack(urldata, rootdir, data) + + return + + def clean(self, urldata, d): + """ + Clean any existing full or partial download + """ + bb.utils.remove(urldata.localpath) + + def try_premirror(self, urldata, d): + """ + Should premirrors be used? + """ + return True + + def checkstatus(self, fetch, urldata, d): + """ + Check the status of a URL + Assumes localpath was called first + """ + logger.info("URL %s could not be checked for status since no method exists.", url) + return True + + def latest_revision(self, ud, d, name): + """ + Look in the cache for the latest revision, if not present ask the SCM. + """ + if not hasattr(self, "_latest_revision"): + raise ParameterError("The fetcher for this URL does not support _latest_revision", url) + + revs = bb.persist_data.persist('BB_URI_HEADREVS', d) + key = self.generate_revision_key(ud, d, name) + try: + return revs[key] + except KeyError: + revs[key] = rev = self._latest_revision(ud, d, name) + return rev + + def sortable_revision(self, ud, d, name): + latest_rev = self._build_revision(ud, d, name) + return True, str(latest_rev) + + def generate_revision_key(self, ud, d, name): + key = self._revision_key(ud, d, name) + return "%s-%s" % (key, d.getVar("PN", True) or "") + +class Fetch(object): + def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): + if localonly and cache: + raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time") + + if len(urls) == 0: + urls = d.getVar("SRC_URI", True).split() + self.urls = urls + self.d = d + self.ud = {} + self.connection_cache = connection_cache + + fn = d.getVar('FILE', True) + if cache and fn and fn in urldata_cache: + self.ud = urldata_cache[fn] + + for url in urls: + if url not in self.ud: + try: + self.ud[url] = FetchData(url, d, localonly) + except NonLocalMethod: + if localonly: + self.ud[url] = None + pass + + if fn and cache: + urldata_cache[fn] = self.ud + + def localpath(self, url): + if url not in self.urls: + self.ud[url] = FetchData(url, self.d) + + self.ud[url].setup_localpath(self.d) + return self.d.expand(self.ud[url].localpath) + + def localpaths(self): + """ + Return a list of the local filenames, assuming successful fetch + """ + local = [] + + for u in self.urls: + ud = self.ud[u] + ud.setup_localpath(self.d) + local.append(ud.localpath) + + return local + + def download(self, urls=None): + """ + Fetch all urls + """ + if not urls: + urls = self.urls + + network = self.d.getVar("BB_NO_NETWORK", True) + premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY", True) == "1") + + for u in urls: + ud = self.ud[u] + ud.setup_localpath(self.d) + m = ud.method + localpath = "" + + if ud.lockfile: + lf = bb.utils.lockfile(ud.lockfile) + + try: + self.d.setVar("BB_NO_NETWORK", network) + + if verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): + localpath = ud.localpath + elif m.try_premirror(ud, self.d): + logger.debug(1, "Trying PREMIRRORS") + mirrors = mirror_from_string(self.d.getVar('PREMIRRORS', True)) + localpath = try_mirrors(self, self.d, ud, mirrors, False) + + if premirroronly: + self.d.setVar("BB_NO_NETWORK", "1") + + os.chdir(self.d.getVar("DL_DIR", True)) + + firsterr = None + verified_stamp = verify_donestamp(ud, self.d) + if not localpath and (not verified_stamp or m.need_update(ud, self.d)): + try: + if not trusted_network(self.d, ud.url): + raise UntrustedUrl(ud.url) + logger.debug(1, "Trying Upstream") + m.download(ud, self.d) + if hasattr(m, "build_mirror_data"): + m.build_mirror_data(ud, self.d) + localpath = ud.localpath + # early checksum verify, so that if checksum mismatched, + # fetcher still have chance to fetch from mirror + update_stamp(ud, self.d) + + except bb.fetch2.NetworkAccess: + raise + + except BBFetchException as e: + if isinstance(e, ChecksumError): + logger.warn("Checksum failure encountered with download of %s - will attempt other sources if available" % u) + logger.debug(1, str(e)) + if os.path.exists(ud.localpath): + rename_bad_checksum(ud, e.checksum) + elif isinstance(e, NoChecksumError): + raise + else: + logger.warn('Failed to fetch URL %s, attempting MIRRORS if available' % u) + logger.debug(1, str(e)) + firsterr = e + # Remove any incomplete fetch + if not verified_stamp: + m.clean(ud, self.d) + logger.debug(1, "Trying MIRRORS") + mirrors = mirror_from_string(self.d.getVar('MIRRORS', True)) + localpath = try_mirrors(self, self.d, ud, mirrors) + + if not localpath or ((not os.path.exists(localpath)) and localpath.find("*") == -1): + if firsterr: + logger.error(str(firsterr)) + raise FetchError("Unable to fetch URL from any source.", u) + + update_stamp(ud, self.d) + + except BBFetchException as e: + if isinstance(e, ChecksumError): + logger.error("Checksum failure fetching %s" % u) + raise + + finally: + if ud.lockfile: + bb.utils.unlockfile(lf) + + def checkstatus(self, urls=None): + """ + Check all urls exist upstream + """ + + if not urls: + urls = self.urls + + for u in urls: + ud = self.ud[u] + ud.setup_localpath(self.d) + m = ud.method + logger.debug(1, "Testing URL %s", u) + # First try checking uri, u, from PREMIRRORS + mirrors = mirror_from_string(self.d.getVar('PREMIRRORS', True)) + ret = try_mirrors(self, self.d, ud, mirrors, True) + if not ret: + # Next try checking from the original uri, u + try: + ret = m.checkstatus(self, ud, self.d) + except: + # Finally, try checking uri, u, from MIRRORS + mirrors = mirror_from_string(self.d.getVar('MIRRORS', True)) + ret = try_mirrors(self, self.d, ud, mirrors, True) + + if not ret: + raise FetchError("URL %s doesn't work" % u, u) + + def unpack(self, root, urls=None): + """ + Check all urls exist upstream + """ + + if not urls: + urls = self.urls + + for u in urls: + ud = self.ud[u] + ud.setup_localpath(self.d) + + if ud.lockfile: + lf = bb.utils.lockfile(ud.lockfile) + + ud.method.unpack(ud, root, self.d) + + if ud.lockfile: + bb.utils.unlockfile(lf) + + def clean(self, urls=None): + """ + Clean files that the fetcher gets or places + """ + + if not urls: + urls = self.urls + + for url in urls: + if url not in self.ud: + self.ud[url] = FetchData(url, d) + ud = self.ud[url] + ud.setup_localpath(self.d) + + if not ud.localfile and ud.localpath is None: + continue + + if ud.lockfile: + lf = bb.utils.lockfile(ud.lockfile) + + ud.method.clean(ud, self.d) + if ud.donestamp: + bb.utils.remove(ud.donestamp) + + if ud.lockfile: + bb.utils.unlockfile(lf) + +class FetchConnectionCache(object): + """ + A class which represents an container for socket connections. + """ + def __init__(self): + self.cache = {} + + def get_connection_name(self, host, port): + return host + ':' + str(port) + + def add_connection(self, host, port, connection): + cn = self.get_connection_name(host, port) + + if cn not in self.cache: + self.cache[cn] = connection + + def get_connection(self, host, port): + connection = None + + cn = self.get_connection_name(host, port) + if cn in self.cache: + connection = self.cache[cn] + + return connection + + def remove_connection(self, host, port): + cn = self.get_connection_name(host, port) + if cn in self.cache: + self.cache[cn].close() + del self.cache[cn] + + def close_connections(self): + for cn in self.cache.keys(): + self.cache[cn].close() + del self.cache[cn] + +from . import cvs +from . import git +from . import gitsm +from . import gitannex +from . import local +from . import svn +from . import wget +from . import ssh +from . import sftp +from . import perforce +from . import bzr +from . import hg +from . import osc +from . import repo +from . import clearcase +from . import npm + +methods.append(local.Local()) +methods.append(wget.Wget()) +methods.append(svn.Svn()) +methods.append(git.Git()) +methods.append(gitsm.GitSM()) +methods.append(gitannex.GitANNEX()) +methods.append(cvs.Cvs()) +methods.append(ssh.SSH()) +methods.append(sftp.SFTP()) +methods.append(perforce.Perforce()) +methods.append(bzr.Bzr()) +methods.append(hg.Hg()) +methods.append(osc.Osc()) +methods.append(repo.Repo()) +methods.append(clearcase.ClearCase()) +methods.append(npm.Npm()) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/bzr.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/bzr.py new file mode 100644 index 000000000..03e9ac461 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/bzr.py @@ -0,0 +1,143 @@ +""" +BitBake 'Fetch' implementation for bzr. + +""" + +# Copyright (C) 2007 Ross Burton +# Copyright (C) 2007 Richard Purdie +# +# Classes for obtaining upstream sources for the +# BitBake build tools. +# Copyright (C) 2003, 2004 Chris Larson +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys +import logging +import bb +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import runfetchcmd +from bb.fetch2 import logger + +class Bzr(FetchMethod): + def supports(self, ud, d): + return ud.type in ['bzr'] + + def urldata_init(self, ud, d): + """ + init bzr specific variable within url data + """ + # Create paths to bzr checkouts + relpath = self._strip_leading_slashes(ud.path) + ud.pkgdir = os.path.join(data.expand('${BZRDIR}', d), ud.host, relpath) + + ud.setup_revisons(d) + + if not ud.revision: + ud.revision = self.latest_revision(ud, d) + + ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d) + + def _buildbzrcommand(self, ud, d, command): + """ + Build up an bzr commandline based on ud + command is "fetch", "update", "revno" + """ + + basecmd = data.expand('${FETCHCMD_bzr}', d) + + proto = ud.parm.get('protocol', 'http') + + bzrroot = ud.host + ud.path + + options = [] + + if command == "revno": + bzrcmd = "%s revno %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot) + else: + if ud.revision: + options.append("-r %s" % ud.revision) + + if command == "fetch": + bzrcmd = "%s branch %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot) + elif command == "update": + bzrcmd = "%s pull %s --overwrite" % (basecmd, " ".join(options)) + else: + raise FetchError("Invalid bzr command %s" % command, ud.url) + + return bzrcmd + + def download(self, ud, d): + """Fetch url""" + + if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK): + bzrcmd = self._buildbzrcommand(ud, d, "update") + logger.debug(1, "BZR Update %s", ud.url) + bb.fetch2.check_network_access(d, bzrcmd, ud.url) + os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path))) + runfetchcmd(bzrcmd, d) + else: + bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True) + bzrcmd = self._buildbzrcommand(ud, d, "fetch") + bb.fetch2.check_network_access(d, bzrcmd, ud.url) + logger.debug(1, "BZR Checkout %s", ud.url) + bb.utils.mkdirhier(ud.pkgdir) + os.chdir(ud.pkgdir) + logger.debug(1, "Running %s", bzrcmd) + runfetchcmd(bzrcmd, d) + + os.chdir(ud.pkgdir) + + scmdata = ud.parm.get("scmdata", "") + if scmdata == "keep": + tar_flags = "" + else: + tar_flags = "--exclude '.bzr' --exclude '.bzrtags'" + + # tar them up to a defined filename + runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d, cleanup = [ud.localpath]) + + def supports_srcrev(self): + return True + + def _revision_key(self, ud, d, name): + """ + Return a unique key for the url + """ + return "bzr:" + ud.pkgdir + + def _latest_revision(self, ud, d, name): + """ + Return the latest upstream revision number + """ + logger.debug(2, "BZR fetcher hitting network for %s", ud.url) + + bb.fetch2.check_network_access(d, self._buildbzrcommand(ud, d, "revno"), ud.url) + + output = runfetchcmd(self._buildbzrcommand(ud, d, "revno"), d, True) + + return output.strip() + + def sortable_revision(self, ud, d, name): + """ + Return a sortable revision number which in our case is the revision number + """ + + return False, self._build_revision(ud, d) + + def _build_revision(self, ud, d): + return ud.revision diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/clearcase.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/clearcase.py new file mode 100644 index 000000000..ba83e7cb6 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/clearcase.py @@ -0,0 +1,263 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' clearcase implementation + +The clearcase fetcher is used to retrieve files from a ClearCase repository. + +Usage in the recipe: + + SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module" + SRCREV = "EXAMPLE_CLEARCASE_TAG" + PV = "${@d.getVar("SRCREV", False).replace("/", "+")}" + +The fetcher uses the rcleartool or cleartool remote client, depending on which one is available. + +Supported SRC_URI options are: + +- vob + (required) The name of the clearcase VOB (with prepending "/") + +- module + The module in the selected VOB (with prepending "/") + + The module and vob parameters are combined to create + the following load rule in the view config spec: + load <vob><module> + +- proto + http or https + +Related variables: + + CCASE_CUSTOM_CONFIG_SPEC + Write a config spec to this variable in your recipe to use it instead + of the default config spec generated by this fetcher. + Please note that the SRCREV loses its functionality if you specify + this variable. SRCREV is still used to label the archive after a fetch, + but it doesn't define what's fetched. + +User credentials: + cleartool: + The login of cleartool is handled by the system. No special steps needed. + + rcleartool: + In order to use rcleartool with authenticated users an `rcleartool login` is + necessary before using the fetcher. +""" +# Copyright (C) 2014 Siemens AG +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import os +import sys +import shutil +import bb +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import runfetchcmd +from bb.fetch2 import logger +from distutils import spawn + +class ClearCase(FetchMethod): + """Class to fetch urls via 'clearcase'""" + def init(self, d): + pass + + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with Clearcase. + """ + return ud.type in ['ccrc'] + + def debug(self, msg): + logger.debug(1, "ClearCase: %s", msg) + + def urldata_init(self, ud, d): + """ + init ClearCase specific variable within url data + """ + ud.proto = "https" + if 'protocol' in ud.parm: + ud.proto = ud.parm['protocol'] + if not ud.proto in ('http', 'https'): + raise fetch2.ParameterError("Invalid protocol type", ud.url) + + ud.vob = '' + if 'vob' in ud.parm: + ud.vob = ud.parm['vob'] + else: + msg = ud.url+": vob must be defined so the fetcher knows what to get." + raise MissingParameterError('vob', msg) + + if 'module' in ud.parm: + ud.module = ud.parm['module'] + else: + ud.module = "" + + ud.basecmd = d.getVar("FETCHCMD_ccrc", True) or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool") + + if data.getVar("SRCREV", d, True) == "INVALID": + raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.") + + ud.label = d.getVar("SRCREV", False) + ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC", True) + + ud.server = "%s://%s%s" % (ud.proto, ud.host, ud.path) + + ud.identifier = "clearcase-%s%s-%s" % ( ud.vob.replace("/", ""), + ud.module.replace("/", "."), + ud.label.replace("/", ".")) + + ud.viewname = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True)) + ud.csname = "%s-config-spec" % (ud.identifier) + ud.ccasedir = os.path.join(data.getVar("DL_DIR", d, True), ud.type) + ud.viewdir = os.path.join(ud.ccasedir, ud.viewname) + ud.configspecfile = os.path.join(ud.ccasedir, ud.csname) + ud.localfile = "%s.tar.gz" % (ud.identifier) + + self.debug("host = %s" % ud.host) + self.debug("path = %s" % ud.path) + self.debug("server = %s" % ud.server) + self.debug("proto = %s" % ud.proto) + self.debug("type = %s" % ud.type) + self.debug("vob = %s" % ud.vob) + self.debug("module = %s" % ud.module) + self.debug("basecmd = %s" % ud.basecmd) + self.debug("label = %s" % ud.label) + self.debug("ccasedir = %s" % ud.ccasedir) + self.debug("viewdir = %s" % ud.viewdir) + self.debug("viewname = %s" % ud.viewname) + self.debug("configspecfile = %s" % ud.configspecfile) + self.debug("localfile = %s" % ud.localfile) + + ud.localfile = os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) + + def _build_ccase_command(self, ud, command): + """ + Build up a commandline based on ud + command is: mkview, setcs, rmview + """ + options = [] + + if "rcleartool" in ud.basecmd: + options.append("-server %s" % ud.server) + + basecmd = "%s %s" % (ud.basecmd, command) + + if command is 'mkview': + if not "rcleartool" in ud.basecmd: + # Cleartool needs a -snapshot view + options.append("-snapshot") + options.append("-tag %s" % ud.viewname) + options.append(ud.viewdir) + + elif command is 'rmview': + options.append("-force") + options.append("%s" % ud.viewdir) + + elif command is 'setcs': + options.append("-overwrite") + options.append(ud.configspecfile) + + else: + raise FetchError("Invalid ccase command %s" % command) + + ccasecmd = "%s %s" % (basecmd, " ".join(options)) + self.debug("ccasecmd = %s" % ccasecmd) + return ccasecmd + + def _write_configspec(self, ud, d): + """ + Create config spec file (ud.configspecfile) for ccase view + """ + config_spec = "" + custom_config_spec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC", d) + if custom_config_spec is not None: + for line in custom_config_spec.split("\\n"): + config_spec += line+"\n" + bb.warn("A custom config spec has been set, SRCREV is only relevant for the tarball name.") + else: + config_spec += "element * CHECKEDOUT\n" + config_spec += "element * %s\n" % ud.label + config_spec += "load %s%s\n" % (ud.vob, ud.module) + + logger.info("Using config spec: \n%s" % config_spec) + + with open(ud.configspecfile, 'w') as f: + f.write(config_spec) + + def _remove_view(self, ud, d): + if os.path.exists(ud.viewdir): + os.chdir(ud.ccasedir) + cmd = self._build_ccase_command(ud, 'rmview'); + logger.info("cleaning up [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname) + bb.fetch2.check_network_access(d, cmd, ud.url) + output = runfetchcmd(cmd, d) + logger.info("rmview output: %s", output) + + def need_update(self, ud, d): + if ("LATEST" in ud.label) or (ud.customspec and "LATEST" in ud.customspec): + ud.identifier += "-%s" % d.getVar("DATETIME",d, True) + return True + if os.path.exists(ud.localpath): + return False + return True + + def supports_srcrev(self): + return True + + def sortable_revision(self, ud, d, name): + return False, ud.identifier + + def download(self, ud, d): + """Fetch url""" + + # Make a fresh view + bb.utils.mkdirhier(ud.ccasedir) + self._write_configspec(ud, d) + cmd = self._build_ccase_command(ud, 'mkview') + logger.info("creating view [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname) + bb.fetch2.check_network_access(d, cmd, ud.url) + try: + runfetchcmd(cmd, d) + except FetchError as e: + if "CRCLI2008E" in e.msg: + raise FetchError("%s\n%s\n" % (e.msg, "Call `rcleartool login` in your console to authenticate to the clearcase server before running bitbake.")) + else: + raise e + + # Set configspec: Setting the configspec effectively fetches the files as defined in the configspec + os.chdir(ud.viewdir) + cmd = self._build_ccase_command(ud, 'setcs'); + logger.info("fetching data [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname) + bb.fetch2.check_network_access(d, cmd, ud.url) + output = runfetchcmd(cmd, d) + logger.info("%s", output) + + # Copy the configspec to the viewdir so we have it in our source tarball later + shutil.copyfile(ud.configspecfile, os.path.join(ud.viewdir, ud.csname)) + + # Clean clearcase meta-data before tar + + runfetchcmd('tar -czf "%s" .' % (ud.localpath), d, cleanup = [ud.localpath]) + + # Clean up so we can create a new view next time + self.clean(ud, d); + + def clean(self, ud, d): + self._remove_view(ud, d) + bb.utils.remove(ud.configspecfile) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/cvs.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/cvs.py new file mode 100644 index 000000000..d27d96f68 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/cvs.py @@ -0,0 +1,171 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +""" + +# Copyright (C) 2003, 2004 Chris Larson +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +#Based on functions from the base bb module, Copyright 2003 Holger Schurig +# + +import os +import logging +import bb +from bb.fetch2 import FetchMethod, FetchError, MissingParameterError, logger +from bb.fetch2 import runfetchcmd + +class Cvs(FetchMethod): + """ + Class to fetch a module or modules from cvs repositories + """ + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with cvs. + """ + return ud.type in ['cvs'] + + def urldata_init(self, ud, d): + if not "module" in ud.parm: + raise MissingParameterError("module", ud.url) + ud.module = ud.parm["module"] + + ud.tag = ud.parm.get('tag', "") + + # Override the default date in certain cases + if 'date' in ud.parm: + ud.date = ud.parm['date'] + elif ud.tag: + ud.date = "" + + norecurse = '' + if 'norecurse' in ud.parm: + norecurse = '_norecurse' + + fullpath = '' + if 'fullpath' in ud.parm: + fullpath = '_fullpath' + + ud.localfile = bb.data.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath), d) + + def need_update(self, ud, d): + if (ud.date == "now"): + return True + if not os.path.exists(ud.localpath): + return True + return False + + def download(self, ud, d): + + method = ud.parm.get('method', 'pserver') + localdir = ud.parm.get('localdir', ud.module) + cvs_port = ud.parm.get('port', '') + + cvs_rsh = None + if method == "ext": + if "rsh" in ud.parm: + cvs_rsh = ud.parm["rsh"] + + if method == "dir": + cvsroot = ud.path + else: + cvsroot = ":" + method + cvsproxyhost = d.getVar('CVS_PROXY_HOST', True) + if cvsproxyhost: + cvsroot += ";proxy=" + cvsproxyhost + cvsproxyport = d.getVar('CVS_PROXY_PORT', True) + if cvsproxyport: + cvsroot += ";proxyport=" + cvsproxyport + cvsroot += ":" + ud.user + if ud.pswd: + cvsroot += ":" + ud.pswd + cvsroot += "@" + ud.host + ":" + cvs_port + ud.path + + options = [] + if 'norecurse' in ud.parm: + options.append("-l") + if ud.date: + # treat YYYYMMDDHHMM specially for CVS + if len(ud.date) == 12: + options.append("-D \"%s %s:%s UTC\"" % (ud.date[0:8], ud.date[8:10], ud.date[10:12])) + else: + options.append("-D \"%s UTC\"" % ud.date) + if ud.tag: + options.append("-r %s" % ud.tag) + + cvsbasecmd = d.getVar("FETCHCMD_cvs", True) + cvscmd = cvsbasecmd + " '-d" + cvsroot + "' co " + " ".join(options) + " " + ud.module + cvsupdatecmd = cvsbasecmd + " '-d" + cvsroot + "' update -d -P " + " ".join(options) + + if cvs_rsh: + cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd) + cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd) + + # create module directory + logger.debug(2, "Fetch: checking for module directory") + pkg = d.getVar('PN', True) + pkgdir = os.path.join(d.getVar('CVSDIR', True), pkg) + moddir = os.path.join(pkgdir, localdir) + if os.access(os.path.join(moddir, 'CVS'), os.R_OK): + logger.info("Update " + ud.url) + bb.fetch2.check_network_access(d, cvsupdatecmd, ud.url) + # update sources there + os.chdir(moddir) + cmd = cvsupdatecmd + else: + logger.info("Fetch " + ud.url) + # check out sources there + bb.utils.mkdirhier(pkgdir) + os.chdir(pkgdir) + logger.debug(1, "Running %s", cvscmd) + bb.fetch2.check_network_access(d, cvscmd, ud.url) + cmd = cvscmd + + runfetchcmd(cmd, d, cleanup = [moddir]) + + if not os.access(moddir, os.R_OK): + raise FetchError("Directory %s was not readable despite sucessful fetch?!" % moddir, ud.url) + + scmdata = ud.parm.get("scmdata", "") + if scmdata == "keep": + tar_flags = "" + else: + tar_flags = "--exclude 'CVS'" + + # tar them up to a defined filename + if 'fullpath' in ud.parm: + os.chdir(pkgdir) + cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir) + else: + os.chdir(moddir) + os.chdir('..') + cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir)) + + runfetchcmd(cmd, d, cleanup = [ud.localpath]) + + def clean(self, ud, d): + """ Clean CVS Files and tarballs """ + + pkg = d.getVar('PN', True) + pkgdir = os.path.join(d.getVar("CVSDIR", True), pkg) + + bb.utils.remove(pkgdir, True) + bb.utils.remove(ud.localpath) + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py new file mode 100644 index 000000000..526668bc2 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py @@ -0,0 +1,435 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' git implementation + +git fetcher support the SRC_URI with format of: +SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..." + +Supported SRC_URI options are: + +- branch + The git branch to retrieve from. The default is "master" + + This option also supports multiple branch fetching, with branches + separated by commas. In multiple branches case, the name option + must have the same number of names to match the branches, which is + used to specify the SRC_REV for the branch + e.g: + SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY" + SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx" + SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY" + +- tag + The git tag to retrieve. The default is "master" + +- protocol + The method to use to access the repository. Common options are "git", + "http", "https", "file", "ssh" and "rsync". The default is "git". + +- rebaseable + rebaseable indicates that the upstream git repo may rebase in the future, + and current revision may disappear from upstream repo. This option will + remind fetcher to preserve local cache carefully for future use. + The default value is "0", set rebaseable=1 for rebaseable git repo. + +- nocheckout + Don't checkout source code when unpacking. set this option for the recipe + who has its own routine to checkout code. + The default is "0", set nocheckout=1 if needed. + +- bareclone + Create a bare clone of the source code and don't checkout the source code + when unpacking. Set this option for the recipe who has its own routine to + checkout code and tracking branch requirements. + The default is "0", set bareclone=1 if needed. + +- nobranch + Don't check the SHA validation for branch. set this option for the recipe + referring to commit which is valid in tag instead of branch. + The default is "0", set nobranch=1 if needed. + +""" + +#Copyright (C) 2005 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import errno +import os +import re +import bb +import errno +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import runfetchcmd +from bb.fetch2 import logger + +class Git(FetchMethod): + """Class to fetch a module or modules from git repositories""" + def init(self, d): + pass + + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with git. + """ + return ud.type in ['git'] + + def supports_checksum(self, urldata): + return False + + def urldata_init(self, ud, d): + """ + init git specific variable within url data + so that the git method like latest_revision() can work + """ + if 'protocol' in ud.parm: + ud.proto = ud.parm['protocol'] + elif not ud.host: + ud.proto = 'file' + else: + ud.proto = "git" + + if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): + raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) + + ud.nocheckout = ud.parm.get("nocheckout","0") == "1" + + ud.rebaseable = ud.parm.get("rebaseable","0") == "1" + + ud.nobranch = ud.parm.get("nobranch","0") == "1" + + # bareclone implies nocheckout + ud.bareclone = ud.parm.get("bareclone","0") == "1" + if ud.bareclone: + ud.nocheckout = 1 + + ud.unresolvedrev = {} + branches = ud.parm.get("branch", "master").split(',') + if len(branches) != len(ud.names): + raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) + ud.branches = {} + for name in ud.names: + branch = branches[ud.names.index(name)] + ud.branches[name] = branch + ud.unresolvedrev[name] = branch + + ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git -c core.fsyncobjectfiles=0" + + ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable + + ud.setup_revisons(d) + + for name in ud.names: + # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one + if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): + if ud.revisions[name]: + ud.unresolvedrev[name] = ud.revisions[name] + ud.revisions[name] = self.latest_revision(ud, d, name) + + gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.')) + if gitsrcname.startswith('.'): + gitsrcname = gitsrcname[1:] + + # for rebaseable git repo, it is necessary to keep mirror tar ball + # per revision, so that even the revision disappears from the + # upstream repo in the future, the mirror will remain intact and still + # contains the revision + if ud.rebaseable: + for name in ud.names: + gitsrcname = gitsrcname + '_' + ud.revisions[name] + ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname) + ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) + gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/") + ud.clonedir = os.path.join(gitdir, gitsrcname) + + ud.localfile = ud.clonedir + + def localpath(self, ud, d): + return ud.clonedir + + def need_update(self, ud, d): + if not os.path.exists(ud.clonedir): + return True + os.chdir(ud.clonedir) + for name in ud.names: + if not self._contains_ref(ud, d, name): + return True + if ud.write_tarballs and not os.path.exists(ud.fullmirror): + return True + return False + + def try_premirror(self, ud, d): + # If we don't do this, updating an existing checkout with only premirrors + # is not possible + if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None: + return True + if os.path.exists(ud.clonedir): + return False + return True + + def download(self, ud, d): + """Fetch url""" + + # If the checkout doesn't exist and the mirror tarball does, extract it + if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): + bb.utils.mkdirhier(ud.clonedir) + os.chdir(ud.clonedir) + runfetchcmd("tar -xzf %s" % (ud.fullmirror), d) + + repourl = self._get_repo_url(ud) + + # If the repo still doesn't exist, fallback to cloning it + if not os.path.exists(ud.clonedir): + # We do this since git will use a "-l" option automatically for local urls where possible + if repourl.startswith("file://"): + repourl = repourl[7:] + clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir) + if ud.proto.lower() != 'file': + bb.fetch2.check_network_access(d, clone_cmd) + runfetchcmd(clone_cmd, d) + + os.chdir(ud.clonedir) + # Update the checkout if needed + needupdate = False + for name in ud.names: + if not self._contains_ref(ud, d, name): + needupdate = True + if needupdate: + try: + runfetchcmd("%s remote rm origin" % ud.basecmd, d) + except bb.fetch2.FetchError: + logger.debug(1, "No Origin") + + runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d) + fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl) + if ud.proto.lower() != 'file': + bb.fetch2.check_network_access(d, fetch_cmd, ud.url) + runfetchcmd(fetch_cmd, d) + runfetchcmd("%s prune-packed" % ud.basecmd, d) + runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) + try: + os.unlink(ud.fullmirror) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + os.chdir(ud.clonedir) + for name in ud.names: + if not self._contains_ref(ud, d, name): + raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) + + def build_mirror_data(self, ud, d): + # Generate a mirror tarball if needed + if ud.write_tarballs and not os.path.exists(ud.fullmirror): + # it's possible that this symlink points to read-only filesystem with PREMIRROR + if os.path.islink(ud.fullmirror): + os.unlink(ud.fullmirror) + + os.chdir(ud.clonedir) + logger.info("Creating tarball of git repository") + runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d) + runfetchcmd("touch %s.done" % (ud.fullmirror), d) + + def unpack(self, ud, destdir, d): + """ unpack the downloaded src to destdir""" + + subdir = ud.parm.get("subpath", "") + if subdir != "": + readpathspec = ":%s" % (subdir) + def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/')) + else: + readpathspec = "" + def_destsuffix = "git/" + + destsuffix = ud.parm.get("destsuffix", def_destsuffix) + destdir = ud.destdir = os.path.join(destdir, destsuffix) + if os.path.exists(destdir): + bb.utils.prunedir(destdir) + + cloneflags = "-s -n" + if ud.bareclone: + cloneflags += " --mirror" + + runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, ud.clonedir, destdir), d) + os.chdir(destdir) + repourl = self._get_repo_url(ud) + runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d) + if not ud.nocheckout: + if subdir != "": + runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d) + runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d) + elif not ud.nobranch: + branchname = ud.branches[ud.names[0]] + runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ + ud.revisions[ud.names[0]]), d) + runfetchcmd("%s branch --set-upstream %s origin/%s" % (ud.basecmd, branchname, \ + branchname), d) + else: + runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d) + + return True + + def clean(self, ud, d): + """ clean the git directory """ + + bb.utils.remove(ud.localpath, True) + bb.utils.remove(ud.fullmirror) + bb.utils.remove(ud.fullmirror + ".done") + + def supports_srcrev(self): + return True + + def _contains_ref(self, ud, d, name): + cmd = "" + if ud.nobranch: + cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( + ud.basecmd, ud.revisions[name]) + else: + cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( + ud.basecmd, ud.revisions[name], ud.branches[name]) + try: + output = runfetchcmd(cmd, d, quiet=True) + except bb.fetch2.FetchError: + return False + if len(output.split()) > 1: + raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) + return output.split()[0] != "0" + + def _get_repo_url(self, ud): + """ + Return the repository URL + """ + if ud.user: + username = ud.user + '@' + else: + username = "" + return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path) + + def _revision_key(self, ud, d, name): + """ + Return a unique key for the url + """ + return "git:" + ud.host + ud.path.replace('/', '.') + ud.unresolvedrev[name] + + def _lsremote(self, ud, d, search): + """ + Run git ls-remote with the specified search string + """ + repourl = self._get_repo_url(ud) + cmd = "%s ls-remote %s %s" % \ + (ud.basecmd, repourl, search) + if ud.proto.lower() != 'file': + bb.fetch2.check_network_access(d, cmd) + output = runfetchcmd(cmd, d, True) + if not output: + raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url) + return output + + def _latest_revision(self, ud, d, name): + """ + Compute the HEAD revision for the url + """ + output = self._lsremote(ud, d, "") + # Tags of the form ^{} may not work, need to fallback to other form + if ud.unresolvedrev[name][:5] == "refs/": + head = ud.unresolvedrev[name] + tag = ud.unresolvedrev[name] + else: + head = "refs/heads/%s" % ud.unresolvedrev[name] + tag = "refs/tags/%s" % ud.unresolvedrev[name] + for s in [head, tag + "^{}", tag]: + for l in output.split('\n'): + if s in l: + return l.split()[0] + raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ + (ud.unresolvedrev[name], ud.host+ud.path)) + + def latest_versionstring(self, ud, d): + """ + Compute the latest release name like "x.y.x" in "x.y.x+gitHASH" + by searching through the tags output of ls-remote, comparing + versions and returning the highest match. + """ + pupver = ('', '') + + tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX', True) or "(?P<pver>([0-9][\.|_]?)+)") + try: + output = self._lsremote(ud, d, "refs/tags/*") + except bb.fetch2.FetchError or bb.fetch2.NetworkAccess: + return pupver + + verstring = "" + revision = "" + for line in output.split("\n"): + if not line: + break + + tag_head = line.split("/")[-1] + # Ignore non-released branches + m = re.search("(alpha|beta|rc|final)+", tag_head) + if m: + continue + + # search for version in the line + tag = tagregex.search(tag_head) + if tag == None: + continue + + tag = tag.group('pver') + tag = tag.replace("_", ".") + + if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: + continue + + verstring = tag + revision = line.split()[0] + pupver = (verstring, revision) + + return pupver + + def _build_revision(self, ud, d, name): + return ud.revisions[name] + + def gitpkgv_revision(self, ud, d, name): + """ + Return a sortable revision number by counting commits in the history + Based on gitpkgv.bblass in meta-openembedded + """ + rev = self._build_revision(ud, d, name) + localpath = ud.localpath + rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) + if not os.path.exists(localpath): + commits = None + else: + if not os.path.exists(rev_file) or not os.path.getsize(rev_file): + from pipes import quote + commits = bb.fetch2.runfetchcmd( + "git rev-list %s -- | wc -l" % (quote(rev)), + d, quiet=True).strip().lstrip('0') + if commits: + open(rev_file, "w").write("%d\n" % int(commits)) + else: + commits = open(rev_file, "r").readline(128).strip() + if commits: + return False, "%s+%s" % (commits, rev[:7]) + else: + return True, str(rev) + + def checkstatus(self, fetch, ud, d): + try: + self._lsremote(ud, d, "") + return True + except FetchError: + return False diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitannex.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitannex.py new file mode 100644 index 000000000..0f3789745 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitannex.py @@ -0,0 +1,76 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' git annex implementation +""" + +# Copyright (C) 2014 Otavio Salvador +# Copyright (C) 2014 O.S. Systems Software LTDA. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import bb +from bb import data +from bb.fetch2.git import Git +from bb.fetch2 import runfetchcmd +from bb.fetch2 import logger + +class GitANNEX(Git): + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with git. + """ + return ud.type in ['gitannex'] + + def uses_annex(self, ud, d): + for name in ud.names: + try: + runfetchcmd("%s rev-list git-annex" % (ud.basecmd), d, quiet=True) + return True + except bb.fetch.FetchError: + pass + + return False + + def update_annex(self, ud, d): + try: + runfetchcmd("%s annex get --all" % (ud.basecmd), d, quiet=True) + except bb.fetch.FetchError: + return False + runfetchcmd("chmod u+w -R %s/annex" % (ud.clonedir), d, quiet=True) + + return True + + def download(self, ud, d): + Git.download(self, ud, d) + + os.chdir(ud.clonedir) + annex = self.uses_annex(ud, d) + if annex: + self.update_annex(ud, d) + + def unpack(self, ud, destdir, d): + Git.unpack(self, ud, destdir, d) + + os.chdir(ud.destdir) + try: + runfetchcmd("%s annex sync" % (ud.basecmd), d) + except bb.fetch.FetchError: + pass + + annex = self.uses_annex(ud, d) + if annex: + runfetchcmd("%s annex get" % (ud.basecmd), d) + runfetchcmd("chmod u+w -R %s/.git/annex" % (ud.destdir), d, quiet=True) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py new file mode 100644 index 000000000..752f1d3c1 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py @@ -0,0 +1,134 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' git submodules implementation + +Inherits from and extends the Git fetcher to retrieve submodules of a git repository +after cloning. + +SRC_URI = "gitsm://<see Git fetcher for syntax>" + +See the Git fetcher, git://, for usage documentation. + +NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. + +""" + +# Copyright (C) 2013 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import bb +from bb import data +from bb.fetch2.git import Git +from bb.fetch2 import runfetchcmd +from bb.fetch2 import logger + +class GitSM(Git): + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with git. + """ + return ud.type in ['gitsm'] + + def uses_submodules(self, ud, d): + for name in ud.names: + try: + runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True) + return True + except bb.fetch.FetchError: + pass + return False + + def _set_relative_paths(self, repopath): + """ + Fix submodule paths to be relative instead of absolute, + so that when we move the repo it doesn't break + (In Git 1.7.10+ this is done automatically) + """ + submodules = [] + with open(os.path.join(repopath, '.gitmodules'), 'r') as f: + for line in f.readlines(): + if line.startswith('[submodule'): + submodules.append(line.split('"')[1]) + + for module in submodules: + repo_conf = os.path.join(repopath, module, '.git') + if os.path.exists(repo_conf): + with open(repo_conf, 'r') as f: + lines = f.readlines() + newpath = '' + for i, line in enumerate(lines): + if line.startswith('gitdir:'): + oldpath = line.split(': ')[-1].rstrip() + if oldpath.startswith('/'): + newpath = '../' * (module.count('/') + 1) + '.git/modules/' + module + lines[i] = 'gitdir: %s\n' % newpath + break + if newpath: + with open(repo_conf, 'w') as f: + for line in lines: + f.write(line) + + repo_conf2 = os.path.join(repopath, '.git', 'modules', module, 'config') + if os.path.exists(repo_conf2): + with open(repo_conf2, 'r') as f: + lines = f.readlines() + newpath = '' + for i, line in enumerate(lines): + if line.lstrip().startswith('worktree = '): + oldpath = line.split(' = ')[-1].rstrip() + if oldpath.startswith('/'): + newpath = '../' * (module.count('/') + 3) + module + lines[i] = '\tworktree = %s\n' % newpath + break + if newpath: + with open(repo_conf2, 'w') as f: + for line in lines: + f.write(line) + + def update_submodules(self, ud, d): + # We have to convert bare -> full repo, do the submodule bit, then convert back + tmpclonedir = ud.clonedir + ".tmp" + gitdir = tmpclonedir + os.sep + ".git" + bb.utils.remove(tmpclonedir, True) + os.mkdir(tmpclonedir) + os.rename(ud.clonedir, gitdir) + runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d) + os.chdir(tmpclonedir) + runfetchcmd(ud.basecmd + " reset --hard", d) + runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d) + runfetchcmd(ud.basecmd + " submodule update --init --recursive", d) + self._set_relative_paths(tmpclonedir) + runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d) + os.rename(gitdir, ud.clonedir,) + bb.utils.remove(tmpclonedir, True) + + def download(self, ud, d): + Git.download(self, ud, d) + + os.chdir(ud.clonedir) + submodules = self.uses_submodules(ud, d) + if submodules: + self.update_submodules(ud, d) + + def unpack(self, ud, destdir, d): + Git.unpack(self, ud, destdir, d) + + os.chdir(ud.destdir) + submodules = self.uses_submodules(ud, d) + if submodules: + runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d) + runfetchcmd(ud.basecmd + " submodule update --init --recursive", d) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/hg.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/hg.py new file mode 100644 index 000000000..3b743ff51 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/hg.py @@ -0,0 +1,278 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementation for mercurial DRCS (hg). + +""" + +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2004 Marcin Juszkiewicz +# Copyright (C) 2007 Robert Schuster +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import os +import sys +import logging +import bb +import errno +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import MissingParameterError +from bb.fetch2 import runfetchcmd +from bb.fetch2 import logger + +class Hg(FetchMethod): + """Class to fetch from mercurial repositories""" + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with mercurial. + """ + return ud.type in ['hg'] + + def supports_checksum(self, urldata): + """ + Don't require checksums for local archives created from + repository checkouts. + """ + return False + + def urldata_init(self, ud, d): + """ + init hg specific variable within url data + """ + if not "module" in ud.parm: + raise MissingParameterError('module', ud.url) + + ud.module = ud.parm["module"] + + if 'protocol' in ud.parm: + ud.proto = ud.parm['protocol'] + elif not ud.host: + ud.proto = 'file' + else: + ud.proto = "hg" + + ud.setup_revisons(d) + + if 'rev' in ud.parm: + ud.revision = ud.parm['rev'] + elif not ud.revision: + ud.revision = self.latest_revision(ud, d) + + # Create paths to mercurial checkouts + hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \ + ud.host, ud.path.replace('/', '.')) + ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname + ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) + + hgdir = d.getVar("HGDIR", True) or (d.getVar("DL_DIR", True) + "/hg/") + ud.pkgdir = os.path.join(hgdir, hgsrcname) + ud.moddir = os.path.join(ud.pkgdir, ud.module) + ud.localfile = ud.moddir + ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or "/usr/bin/env hg" + + ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS", True) + + def need_update(self, ud, d): + revTag = ud.parm.get('rev', 'tip') + if revTag == "tip": + return True + if not os.path.exists(ud.localpath): + return True + return False + + def try_premirror(self, ud, d): + # If we don't do this, updating an existing checkout with only premirrors + # is not possible + if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None: + return True + if os.path.exists(ud.moddir): + return False + return True + + def _buildhgcommand(self, ud, d, command): + """ + Build up an hg commandline based on ud + command is "fetch", "update", "info" + """ + + proto = ud.parm.get('protocol', 'http') + + host = ud.host + if proto == "file": + host = "/" + ud.host = "localhost" + + if not ud.user: + hgroot = host + ud.path + else: + if ud.pswd: + hgroot = ud.user + ":" + ud.pswd + "@" + host + ud.path + else: + hgroot = ud.user + "@" + host + ud.path + + if command == "info": + return "%s identify -i %s://%s/%s" % (ud.basecmd, proto, hgroot, ud.module) + + options = []; + + # Don't specify revision for the fetch; clone the entire repo. + # This avoids an issue if the specified revision is a tag, because + # the tag actually exists in the specified revision + 1, so it won't + # be available when used in any successive commands. + if ud.revision and command != "fetch": + options.append("-r %s" % ud.revision) + + if command == "fetch": + if ud.user and ud.pswd: + cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" clone %s %s://%s/%s %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options), proto, hgroot, ud.module, ud.module) + else: + cmd = "%s clone %s %s://%s/%s %s" % (ud.basecmd, " ".join(options), proto, hgroot, ud.module, ud.module) + elif command == "pull": + # do not pass options list; limiting pull to rev causes the local + # repo not to contain it and immediately following "update" command + # will crash + if ud.user and ud.pswd: + cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (ud.basecmd, ud.user, ud.pswd, proto) + else: + cmd = "%s pull" % (ud.basecmd) + elif command == "update": + if ud.user and ud.pswd: + cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options)) + else: + cmd = "%s update -C %s" % (ud.basecmd, " ".join(options)) + else: + raise FetchError("Invalid hg command %s" % command, ud.url) + + return cmd + + def download(self, ud, d): + """Fetch url""" + + logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") + + # If the checkout doesn't exist and the mirror tarball does, extract it + if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror): + bb.utils.mkdirhier(ud.pkgdir) + os.chdir(ud.pkgdir) + runfetchcmd("tar -xzf %s" % (ud.fullmirror), d) + + if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): + # Found the source, check whether need pull + updatecmd = self._buildhgcommand(ud, d, "update") + os.chdir(ud.moddir) + logger.debug(1, "Running %s", updatecmd) + try: + runfetchcmd(updatecmd, d) + except bb.fetch2.FetchError: + # Runnning pull in the repo + pullcmd = self._buildhgcommand(ud, d, "pull") + logger.info("Pulling " + ud.url) + # update sources there + os.chdir(ud.moddir) + logger.debug(1, "Running %s", pullcmd) + bb.fetch2.check_network_access(d, pullcmd, ud.url) + runfetchcmd(pullcmd, d) + try: + os.unlink(ud.fullmirror) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + + # No source found, clone it. + if not os.path.exists(ud.moddir): + fetchcmd = self._buildhgcommand(ud, d, "fetch") + logger.info("Fetch " + ud.url) + # check out sources there + bb.utils.mkdirhier(ud.pkgdir) + os.chdir(ud.pkgdir) + logger.debug(1, "Running %s", fetchcmd) + bb.fetch2.check_network_access(d, fetchcmd, ud.url) + runfetchcmd(fetchcmd, d) + + # Even when we clone (fetch), we still need to update as hg's clone + # won't checkout the specified revision if its on a branch + updatecmd = self._buildhgcommand(ud, d, "update") + os.chdir(ud.moddir) + logger.debug(1, "Running %s", updatecmd) + runfetchcmd(updatecmd, d) + + def clean(self, ud, d): + """ Clean the hg dir """ + + bb.utils.remove(ud.localpath, True) + bb.utils.remove(ud.fullmirror) + bb.utils.remove(ud.fullmirror + ".done") + + def supports_srcrev(self): + return True + + def _latest_revision(self, ud, d, name): + """ + Compute tip revision for the url + """ + bb.fetch2.check_network_access(d, self._buildhgcommand(ud, d, "info")) + output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d) + return output.strip() + + def _build_revision(self, ud, d, name): + return ud.revision + + def _revision_key(self, ud, d, name): + """ + Return a unique key for the url + """ + return "hg:" + ud.moddir + + def build_mirror_data(self, ud, d): + # Generate a mirror tarball if needed + if ud.write_tarballs == "1" and not os.path.exists(ud.fullmirror): + # it's possible that this symlink points to read-only filesystem with PREMIRROR + if os.path.islink(ud.fullmirror): + os.unlink(ud.fullmirror) + + os.chdir(ud.pkgdir) + logger.info("Creating tarball of hg repository") + runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d) + runfetchcmd("touch %s.done" % (ud.fullmirror), d) + + def localpath(self, ud, d): + return ud.pkgdir + + def unpack(self, ud, destdir, d): + """ + Make a local clone or export for the url + """ + + revflag = "-r %s" % ud.revision + subdir = ud.parm.get("destsuffix", ud.module) + codir = "%s/%s" % (destdir, subdir) + + scmdata = ud.parm.get("scmdata", "") + if scmdata != "nokeep": + if not os.access(os.path.join(codir, '.hg'), os.R_OK): + logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'") + runfetchcmd("%s init %s" % (ud.basecmd, codir), d) + logger.debug(2, "Unpack: updating source in '" + codir + "'") + os.chdir(codir) + runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d) + runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d) + else: + logger.debug(2, "Unpack: extracting source to '" + codir + "'") + os.chdir(ud.moddir) + runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/local.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/local.py new file mode 100644 index 000000000..303a52b63 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/local.py @@ -0,0 +1,129 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +""" + +# Copyright (C) 2003, 2004 Chris Larson +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import os +import urllib +import bb +import bb.utils +from bb import data +from bb.fetch2 import FetchMethod, FetchError +from bb.fetch2 import logger + +class Local(FetchMethod): + def supports(self, urldata, d): + """ + Check to see if a given url represents a local fetch. + """ + return urldata.type in ['file'] + + def urldata_init(self, ud, d): + # We don't set localfile as for this fetcher the file is already local! + ud.decodedurl = urllib.unquote(ud.url.split("://")[1].split(";")[0]) + ud.basename = os.path.basename(ud.decodedurl) + ud.basepath = ud.decodedurl + ud.needdonestamp = False + return + + def localpath(self, urldata, d): + """ + Return the local filename of a given url assuming a successful fetch. + """ + return self.localpaths(urldata, d)[-1] + + def localpaths(self, urldata, d): + """ + Return the local filename of a given url assuming a successful fetch. + """ + searched = [] + path = urldata.decodedurl + newpath = path + if path[0] == "/": + return [path] + filespath = data.getVar('FILESPATH', d, True) + if filespath: + logger.debug(2, "Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":")))) + newpath, hist = bb.utils.which(filespath, path, history=True) + searched.extend(hist) + if not newpath: + filesdir = data.getVar('FILESDIR', d, True) + if filesdir: + logger.debug(2, "Searching for %s in path: %s" % (path, filesdir)) + newpath = os.path.join(filesdir, path) + searched.append(newpath) + if (not newpath or not os.path.exists(newpath)) and path.find("*") != -1: + # For expressions using '*', best we can do is take the first directory in FILESPATH that exists + newpath, hist = bb.utils.which(filespath, ".", history=True) + searched.extend(hist) + logger.debug(2, "Searching for %s in path: %s" % (path, newpath)) + return searched + if not os.path.exists(newpath): + dldirfile = os.path.join(d.getVar("DL_DIR", True), path) + logger.debug(2, "Defaulting to %s for %s" % (dldirfile, path)) + bb.utils.mkdirhier(os.path.dirname(dldirfile)) + searched.append(dldirfile) + return searched + return searched + + def need_update(self, ud, d): + if ud.url.find("*") != -1: + return False + if os.path.exists(ud.localpath): + return False + return True + + def download(self, urldata, d): + """Fetch urls (no-op for Local method)""" + # no need to fetch local files, we'll deal with them in place. + if self.supports_checksum(urldata) and not os.path.exists(urldata.localpath): + locations = [] + filespath = data.getVar('FILESPATH', d, True) + if filespath: + locations = filespath.split(":") + filesdir = data.getVar('FILESDIR', d, True) + if filesdir: + locations.append(filesdir) + locations.append(d.getVar("DL_DIR", True)) + + msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n " + "\n ".join(locations) + raise FetchError(msg) + + return True + + def checkstatus(self, fetch, urldata, d): + """ + Check the status of the url + """ + if urldata.localpath.find("*") != -1: + logger.info("URL %s looks like a glob and was therefore not checked.", urldata.url) + return True + if os.path.exists(urldata.localpath): + return True + return False + + def clean(self, urldata, d): + return + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py new file mode 100644 index 000000000..e8d9b1109 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py @@ -0,0 +1,284 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' NPM implementation + +The NPM fetcher is used to retrieve files from the npmjs repository + +Usage in the recipe: + + SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}" + Suported SRC_URI options are: + + - name + - version + + npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz would become npm://registry.npmjs.org;name=${PN};ver=${PV} + The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done + +""" + +import os +import sys +import urllib +import json +import subprocess +import signal +import bb +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import ChecksumError +from bb.fetch2 import runfetchcmd +from bb.fetch2 import logger +from bb.fetch2 import UnpackError +from bb.fetch2 import ParameterError +from distutils import spawn + +def subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + # SIGPIPE errors are known issues with gzip/bash + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +class Npm(FetchMethod): + + """Class to fetch urls via 'npm'""" + def init(self, d): + pass + + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with npm + """ + return ud.type in ['npm'] + + def debug(self, msg): + logger.debug(1, "NpmFetch: %s", msg) + + def clean(self, ud, d): + logger.debug(2, "Calling cleanup %s" % ud.pkgname) + bb.utils.remove(ud.localpath, False) + bb.utils.remove(ud.pkgdatadir, True) + bb.utils.remove(ud.fullmirror, False) + + def urldata_init(self, ud, d): + """ + init NPM specific variable within url data + """ + if 'downloadfilename' in ud.parm: + ud.basename = ud.parm['downloadfilename'] + else: + ud.basename = os.path.basename(ud.path) + + # can't call it ud.name otherwise fetcher base class will start doing sha1stuff + # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps + ud.pkgname = ud.parm.get("name", None) + if not ud.pkgname: + raise ParameterError("NPM fetcher requires a name parameter", ud.url) + ud.version = ud.parm.get("version", None) + if not ud.version: + raise ParameterError("NPM fetcher requires a version parameter", ud.url) + ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version) + ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0] + prefixdir = "npm/%s" % ud.pkgname + ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir) + if not os.path.exists(ud.pkgdatadir): + bb.utils.mkdirhier(ud.pkgdatadir) + ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest) + + self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate " + self.basecmd += " --directory-prefix=%s " % prefixdir + + ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") + ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) + ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) + + def need_update(self, ud, d): + if os.path.exists(ud.localpath): + return False + return True + + def _runwget(self, ud, d, command, quiet): + logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) + bb.fetch2.check_network_access(d, command) + runfetchcmd(command, d, quiet) + + def _unpackdep(self, ud, pkg, data, destdir, dldir, d): + file = data[pkg]['tgz'] + logger.debug(2, "file to extract is %s" % file) + if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): + cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file) + else: + bb.fatal("NPM package %s downloaded not a tarball!" % file) + + # Change to subdir before executing command + save_cwd = os.getcwd() + if not os.path.exists(destdir): + os.makedirs(destdir) + os.chdir(destdir) + path = d.getVar('PATH', True) + if path: + cmd = "PATH=\"%s\" %s" % (path, cmd) + bb.note("Unpacking %s to %s/" % (file, os.getcwd())) + ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) + os.chdir(save_cwd) + + if ret != 0: + raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) + + if 'deps' not in data[pkg]: + return + for dep in data[pkg]['deps']: + self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d) + + + def unpack(self, ud, destdir, d): + dldir = d.getVar("DL_DIR", True) + depdumpfile = "%s-%s.deps.json" % (ud.pkgname, ud.version) + with open("%s/npm/%s" % (dldir, depdumpfile)) as datafile: + workobj = json.load(datafile) + dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname) + + self._unpackdep(ud, ud.pkgname, workobj, "%s/npmpkg" % destdir, dldir, d) + + def _parse_view(self, output): + ''' + Parse the output of npm view --json; the last JSON result + is assumed to be the one that we're interested in. + ''' + pdata = None + outdeps = {} + datalines = [] + bracelevel = 0 + for line in output.splitlines(): + if bracelevel: + datalines.append(line) + elif '{' in line: + datalines = [] + datalines.append(line) + bracelevel = bracelevel + line.count('{') - line.count('}') + if datalines: + pdata = json.loads('\n'.join(datalines)) + return pdata + + def _getdependencies(self, pkg, data, version, d, ud, optional=False): + pkgfullname = pkg + if version != '*' and not '/' in version: + pkgfullname += "@'%s'" % version + logger.debug(2, "Calling getdeps on %s" % pkg) + fetchcmd = "npm view %s --json --registry %s" % (pkgfullname, ud.registry) + output = runfetchcmd(fetchcmd, d, True) + pdata = self._parse_view(output) + if not pdata: + raise FetchError("The command '%s' returned no output" % fetchcmd) + if optional: + pkg_os = pdata.get('os', None) + if pkg_os: + if not isinstance(pkg_os, list): + pkg_os = [pkg_os] + if 'linux' not in pkg_os or '!linux' in pkg_os: + logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg) + return + #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile)) + outputurl = pdata['dist']['tarball'] + data[pkg] = {} + data[pkg]['tgz'] = os.path.basename(outputurl) + self._runwget(ud, d, "%s %s" % (self.basecmd, outputurl), False) + + dependencies = pdata.get('dependencies', {}) + optionalDependencies = pdata.get('optionalDependencies', {}) + depsfound = {} + optdepsfound = {} + data[pkg]['deps'] = {} + for dep in dependencies: + if dep in optionalDependencies: + optdepsfound[dep] = dependencies[dep] + else: + depsfound[dep] = dependencies[dep] + for dep, version in optdepsfound.iteritems(): + self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True) + for dep, version in depsfound.iteritems(): + self._getdependencies(dep, data[pkg]['deps'], version, d, ud) + + def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest): + logger.debug(2, "NPM shrinkwrap file is %s" % data) + outputurl = "invalid" + if ('resolved' not in data) or (not data['resolved'].startswith('http')): + # will be the case for ${PN} + fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry) + logger.debug(2, "Found this matching URL: %s" % str(fetchcmd)) + outputurl = runfetchcmd(fetchcmd, d, True) + else: + outputurl = data['resolved'] + self._runwget(ud, d, "%s %s" % (self.basecmd, outputurl), False) + manifest[pkg] = {} + manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip() + manifest[pkg]['deps'] = {} + + if pkg in lockdown: + sha1_expected = lockdown[pkg][version] + sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz'])) + if sha1_expected != sha1_data: + msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected) + raise ChecksumError('Checksum mismatch!%s' % msg) + else: + logger.debug(2, "No lockdown data for %s@%s" % (pkg, version)) + + if 'dependencies' in data: + for obj in data['dependencies']: + logger.debug(2, "Found dep is %s" % str(obj)) + self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps']) + + def download(self, ud, d): + """Fetch url""" + jsondepobj = {} + shrinkobj = {} + lockdown = {} + + if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror): + dest = d.getVar("DL_DIR", True) + bb.utils.mkdirhier(dest) + save_cwd = os.getcwd() + os.chdir(dest) + runfetchcmd("tar -xJf %s" % (ud.fullmirror), d) + os.chdir(save_cwd) + return + + shwrf = d.getVar('NPM_SHRINKWRAP', True) + logger.debug(2, "NPM shrinkwrap file is %s" % shwrf) + try: + with open(shwrf) as datafile: + shrinkobj = json.load(datafile) + except: + logger.warn('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname) + lckdf = d.getVar('NPM_LOCKDOWN', True) + logger.debug(2, "NPM lockdown file is %s" % lckdf) + try: + with open(lckdf) as datafile: + lockdown = json.load(datafile) + except: + logger.warn('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname) + + if ('name' not in shrinkobj): + self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud) + else: + self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj) + + with open(ud.localpath, 'w') as outfile: + json.dump(jsondepobj, outfile) + + def build_mirror_data(self, ud, d): + # Generate a mirror tarball if needed + if ud.write_tarballs and not os.path.exists(ud.fullmirror): + # it's possible that this symlink points to read-only filesystem with PREMIRROR + if os.path.islink(ud.fullmirror): + os.unlink(ud.fullmirror) + + save_cwd = os.getcwd() + os.chdir(d.getVar("DL_DIR", True)) + logger.info("Creating tarball of npm data") + runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d) + runfetchcmd("touch %s.done" % (ud.fullmirror), d) + os.chdir(save_cwd) + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/osc.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/osc.py new file mode 100644 index 000000000..d051dfdaf --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/osc.py @@ -0,0 +1,135 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +Bitbake "Fetch" implementation for osc (Opensuse build service client). +Based on the svn "Fetch" implementation. + +""" + +import os +import sys +import logging +import bb +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import MissingParameterError +from bb.fetch2 import runfetchcmd + +class Osc(FetchMethod): + """Class to fetch a module or modules from Opensuse build server + repositories.""" + + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with osc. + """ + return ud.type in ['osc'] + + def urldata_init(self, ud, d): + if not "module" in ud.parm: + raise MissingParameterError('module', ud.url) + + ud.module = ud.parm["module"] + + # Create paths to osc checkouts + relpath = self._strip_leading_slashes(ud.path) + ud.pkgdir = os.path.join(d.getVar('OSCDIR', True), ud.host) + ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module) + + if 'rev' in ud.parm: + ud.revision = ud.parm['rev'] + else: + pv = d.getVar("PV", False) + rev = bb.fetch2.srcrev_internal_helper(ud, d) + if rev and rev != True: + ud.revision = rev + else: + ud.revision = "" + + ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d) + + def _buildosccommand(self, ud, d, command): + """ + Build up an ocs commandline based on ud + command is "fetch", "update", "info" + """ + + basecmd = data.expand('${FETCHCMD_osc}', d) + + proto = ud.parm.get('protocol', 'ocs') + + options = [] + + config = "-c %s" % self.generate_config(ud, d) + + if ud.revision: + options.append("-r %s" % ud.revision) + + coroot = self._strip_leading_slashes(ud.path) + + if command == "fetch": + osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options)) + elif command == "update": + osccmd = "%s %s up %s" % (basecmd, config, " ".join(options)) + else: + raise FetchError("Invalid osc command %s" % command, ud.url) + + return osccmd + + def download(self, ud, d): + """ + Fetch url + """ + + logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") + + if os.access(os.path.join(d.getVar('OSCDIR', True), ud.path, ud.module), os.R_OK): + oscupdatecmd = self._buildosccommand(ud, d, "update") + logger.info("Update "+ ud.url) + # update sources there + os.chdir(ud.moddir) + logger.debug(1, "Running %s", oscupdatecmd) + bb.fetch2.check_network_access(d, oscupdatecmd, ud.url) + runfetchcmd(oscupdatecmd, d) + else: + oscfetchcmd = self._buildosccommand(ud, d, "fetch") + logger.info("Fetch " + ud.url) + # check out sources there + bb.utils.mkdirhier(ud.pkgdir) + os.chdir(ud.pkgdir) + logger.debug(1, "Running %s", oscfetchcmd) + bb.fetch2.check_network_access(d, oscfetchcmd, ud.url) + runfetchcmd(oscfetchcmd, d) + + os.chdir(os.path.join(ud.pkgdir + ud.path)) + # tar them up to a defined filename + runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d, cleanup = [ud.localpath]) + + def supports_srcrev(self): + return False + + def generate_config(self, ud, d): + """ + Generate a .oscrc to be used for this run. + """ + + config_path = os.path.join(d.getVar('OSCDIR', True), "oscrc") + if (os.path.exists(config_path)): + os.remove(config_path) + + f = open(config_path, 'w') + f.write("[general]\n") + f.write("apisrv = %s\n" % ud.host) + f.write("scheme = http\n") + f.write("su-wrapper = su -c\n") + f.write("build-root = %s\n" % d.getVar('WORKDIR', True)) + f.write("urllist = %s\n" % d.getVar("OSCURLLIST", True)) + f.write("extra-pkgs = gzip\n") + f.write("\n") + f.write("[%s]\n" % ud.host) + f.write("user = %s\n" % ud.parm["user"]) + f.write("pass = %s\n" % ud.parm["pswd"]) + f.close() + + return config_path diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/perforce.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/perforce.py new file mode 100644 index 000000000..3a10c7ca3 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/perforce.py @@ -0,0 +1,187 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +""" + +# Copyright (C) 2003, 2004 Chris Larson +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +from future_builtins import zip +import os +import subprocess +import logging +import bb +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import logger +from bb.fetch2 import runfetchcmd + +class Perforce(FetchMethod): + def supports(self, ud, d): + return ud.type in ['p4'] + + def doparse(url, d): + parm = {} + path = url.split("://")[1] + delim = path.find("@"); + if delim != -1: + (user, pswd, host, port) = path.split('@')[0].split(":") + path = path.split('@')[1] + else: + (host, port) = d.getVar('P4PORT', False).split(':') + user = "" + pswd = "" + + if path.find(";") != -1: + keys=[] + values=[] + plist = path.split(';') + for item in plist: + if item.count('='): + (key, value) = item.split('=') + keys.append(key) + values.append(value) + + parm = dict(zip(keys, values)) + path = "//" + path.split(';')[0] + host += ":%s" % (port) + parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm) + + return host, path, user, pswd, parm + doparse = staticmethod(doparse) + + def getcset(d, depot, host, user, pswd, parm): + p4opt = "" + if "cset" in parm: + return parm["cset"]; + if user: + p4opt += " -u %s" % (user) + if pswd: + p4opt += " -P %s" % (pswd) + if host: + p4opt += " -p %s" % (host) + + p4date = d.getVar("P4DATE", True) + if "revision" in parm: + depot += "#%s" % (parm["revision"]) + elif "label" in parm: + depot += "@%s" % (parm["label"]) + elif p4date: + depot += "@%s" % (p4date) + + p4cmd = d.getVar('FETCHCMD_p4', True) or "p4" + logger.debug(1, "Running %s%s changes -m 1 %s", p4cmd, p4opt, depot) + p4file, errors = bb.process.run("%s%s changes -m 1 %s" % (p4cmd, p4opt, depot)) + cset = p4file.strip() + logger.debug(1, "READ %s", cset) + if not cset: + return -1 + + return cset.split(' ')[1] + getcset = staticmethod(getcset) + + def urldata_init(self, ud, d): + (host, path, user, pswd, parm) = Perforce.doparse(ud.url, d) + + base_path = path.replace('/...', '') + base_path = self._strip_leading_slashes(base_path) + + if "label" in parm: + version = parm["label"] + else: + version = Perforce.getcset(d, path, host, user, pswd, parm) + + ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base_path.replace('/', '.'), version), d) + + def download(self, ud, d): + """ + Fetch urls + """ + + (host, depot, user, pswd, parm) = Perforce.doparse(ud.url, d) + + if depot.find('/...') != -1: + path = depot[:depot.find('/...')] + else: + path = depot[:depot.rfind('/')] + + module = parm.get('module', os.path.basename(path)) + + # Get the p4 command + p4opt = "" + if user: + p4opt += " -u %s" % (user) + + if pswd: + p4opt += " -P %s" % (pswd) + + if host: + p4opt += " -p %s" % (host) + + p4cmd = d.getVar('FETCHCMD_p4', True) or "p4" + + # create temp directory + logger.debug(2, "Fetch: creating temporary directory") + bb.utils.mkdirhier(d.expand('${WORKDIR}')) + mktemp = d.getVar("FETCHCMD_p4mktemp", True) or d.expand("mktemp -d -q '${WORKDIR}/oep4.XXXXXX'") + tmpfile, errors = bb.process.run(mktemp) + tmpfile = tmpfile.strip() + if not tmpfile: + raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", ud.url) + + if "label" in parm: + depot = "%s@%s" % (depot, parm["label"]) + else: + cset = Perforce.getcset(d, depot, host, user, pswd, parm) + depot = "%s@%s" % (depot, cset) + + os.chdir(tmpfile) + logger.info("Fetch " + ud.url) + logger.info("%s%s files %s", p4cmd, p4opt, depot) + p4file, errors = bb.process.run("%s%s files %s" % (p4cmd, p4opt, depot)) + p4file = [f.rstrip() for f in p4file.splitlines()] + + if not p4file: + raise FetchError("Fetch: unable to get the P4 files from %s" % depot, ud.url) + + count = 0 + + for file in p4file: + list = file.split() + + if list[2] == "delete": + continue + + dest = list[0][len(path)+1:] + where = dest.find("#") + + subprocess.call("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module, dest[:where], list[0]), shell=True) + count = count + 1 + + if count == 0: + logger.error() + raise FetchError("Fetch: No files gathered from the P4 fetch", ud.url) + + runfetchcmd("tar -czf %s %s" % (ud.localpath, module), d, cleanup = [ud.localpath]) + # cleanup + bb.utils.prunedir(tmpfile) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/repo.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/repo.py new file mode 100644 index 000000000..21678eb7d --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/repo.py @@ -0,0 +1,98 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake "Fetch" repo (git) implementation + +""" + +# Copyright (C) 2009 Tom Rini <trini@embeddedalley.com> +# +# Based on git.py which is: +#Copyright (C) 2005 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import bb +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import runfetchcmd + +class Repo(FetchMethod): + """Class to fetch a module or modules from repo (git) repositories""" + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with repo. + """ + return ud.type in ["repo"] + + def urldata_init(self, ud, d): + """ + We don"t care about the git rev of the manifests repository, but + we do care about the manifest to use. The default is "default". + We also care about the branch or tag to be used. The default is + "master". + """ + + ud.proto = ud.parm.get('protocol', 'git') + ud.branch = ud.parm.get('branch', 'master') + ud.manifest = ud.parm.get('manifest', 'default.xml') + if not ud.manifest.endswith('.xml'): + ud.manifest += '.xml' + + ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d) + + def download(self, ud, d): + """Fetch url""" + + if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK): + logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath) + return + + gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) + repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo") + codir = os.path.join(repodir, gitsrcname, ud.manifest) + + if ud.user: + username = ud.user + "@" + else: + username = "" + + bb.utils.mkdirhier(os.path.join(codir, "repo")) + os.chdir(os.path.join(codir, "repo")) + if not os.path.exists(os.path.join(codir, "repo", ".repo")): + bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url) + runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d) + + bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url) + runfetchcmd("repo sync", d) + os.chdir(codir) + + scmdata = ud.parm.get("scmdata", "") + if scmdata == "keep": + tar_flags = "" + else: + tar_flags = "--exclude '.repo' --exclude '.git'" + + # Create a cache + runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d) + + def supports_srcrev(self): + return False + + def _build_revision(self, ud, d): + return ud.manifest + + def _want_sortable_revision(self, ud, d): + return False diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/sftp.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/sftp.py new file mode 100644 index 000000000..cb2f753a8 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/sftp.py @@ -0,0 +1,129 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake SFTP Fetch implementation + +Class for fetching files via SFTP. It tries to adhere to the (now +expired) IETF Internet Draft for "Uniform Resource Identifier (URI) +Scheme for Secure File Transfer Protocol (SFTP) and Secure Shell +(SSH)" (SECSH URI). + +It uses SFTP (as to adhere to the SECSH URI specification). It only +supports key based authentication, not password. This class, unlike +the SSH fetcher, does not support fetching a directory tree from the +remote. + + http://tools.ietf.org/html/draft-ietf-secsh-scp-sftp-ssh-uri-04 + https://www.iana.org/assignments/uri-schemes/prov/sftp + https://tools.ietf.org/html/draft-ietf-secsh-filexfer-13 + +Please note that '/' is used as host path seperator, and not ":" +as you may be used to from the scp/sftp commands. You can use a +~ (tilde) to specify a path relative to your home directory. +(The /~user/ syntax, for specyfing a path relative to another +user's home directory is not supported.) Note that the tilde must +still follow the host path seperator ("/"). See exampels below. + +Example SRC_URIs: + +SRC_URI = "sftp://host.example.com/dir/path.file.txt" + +A path relative to your home directory. + +SRC_URI = "sftp://host.example.com/~/dir/path.file.txt" + +You can also specify a username (specyfing password in the +URI is not supported, use SSH keys to authenticate): + +SRC_URI = "sftp://user@host.example.com/dir/path.file.txt" + +""" + +# Copyright (C) 2013, Olof Johansson <olof.johansson@axis.com> +# +# Based in part on bb.fetch2.wget: +# Copyright (C) 2003, 2004 Chris Larson +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import os +import bb +import urllib +import commands +from bb import data +from bb.fetch2 import URI +from bb.fetch2 import FetchMethod +from bb.fetch2 import runfetchcmd + + +class SFTP(FetchMethod): + """Class to fetch urls via 'sftp'""" + + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with sftp. + """ + return ud.type in ['sftp'] + + def recommends_checksum(self, urldata): + return True + + def urldata_init(self, ud, d): + if 'protocol' in ud.parm and ud.parm['protocol'] == 'git': + raise bb.fetch2.ParameterError( + "Invalid protocol - if you wish to fetch from a " + + "git repository using ssh, you need to use the " + + "git:// prefix with protocol=ssh", ud.url) + + if 'downloadfilename' in ud.parm: + ud.basename = ud.parm['downloadfilename'] + else: + ud.basename = os.path.basename(ud.path) + + ud.localfile = data.expand(urllib.unquote(ud.basename), d) + + def download(self, ud, d): + """Fetch urls""" + + urlo = URI(ud.url) + basecmd = 'sftp -oBatchMode=yes' + port = '' + if urlo.port: + port = '-P %d' % urlo.port + urlo.port = None + + dldir = data.getVar('DL_DIR', d, True) + lpath = os.path.join(dldir, ud.localfile) + + user = '' + if urlo.userinfo: + user = urlo.userinfo + '@' + + path = urlo.path + + # Supoprt URIs relative to the user's home directory, with + # the tilde syntax. (E.g. <sftp://example.com/~/foo.diff>). + if path[:3] == '/~/': + path = path[3:] + + remote = '%s%s:%s' % (user, urlo.hostname, path) + + cmd = '%s %s %s %s' % (basecmd, port, commands.mkarg(remote), + commands.mkarg(lpath)) + + bb.fetch2.check_network_access(d, cmd, ud.url) + runfetchcmd(cmd, d) + return True diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/ssh.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/ssh.py new file mode 100644 index 000000000..635578a71 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/ssh.py @@ -0,0 +1,128 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +''' +BitBake 'Fetch' implementations + +This implementation is for Secure Shell (SSH), and attempts to comply with the +IETF secsh internet draft: + http://tools.ietf.org/wg/secsh/draft-ietf-secsh-scp-sftp-ssh-uri/ + + Currently does not support the sftp parameters, as this uses scp + Also does not support the 'fingerprint' connection parameter. + + Please note that '/' is used as host, path separator not ':' as you may + be used to, also '~' can be used to specify user HOME, but again after '/' + + Example SRC_URI: + SRC_URI = "ssh://user@host.example.com/dir/path/file.txt" + SRC_URI = "ssh://user@host.example.com/~/file.txt" +''' + +# Copyright (C) 2006 OpenedHand Ltd. +# +# +# Based in part on svk.py: +# Copyright (C) 2006 Holger Hans Peter Freyther +# Based on svn.py: +# Copyright (C) 2003, 2004 Chris Larson +# Based on functions from the base bb module: +# Copyright 2003 Holger Schurig +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import re, os +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import logger +from bb.fetch2 import runfetchcmd + + +__pattern__ = re.compile(r''' + \s* # Skip leading whitespace + ssh:// # scheme + ( # Optional username/password block + (?P<user>\S+) # username + (:(?P<pass>\S+))? # colon followed by the password (optional) + )? + (?P<cparam>(;[^;]+)*)? # connection parameters block (optional) + @ + (?P<host>\S+?) # non-greedy match of the host + (:(?P<port>[0-9]+))? # colon followed by the port (optional) + / + (?P<path>[^;]+) # path on the remote system, may be absolute or relative, + # and may include the use of '~' to reference the remote home + # directory + (?P<sparam>(;[^;]+)*)? # parameters block (optional) + $ +''', re.VERBOSE) + +class SSH(FetchMethod): + '''Class to fetch a module or modules via Secure Shell''' + + def supports(self, urldata, d): + return __pattern__.match(urldata.url) != None + + def supports_checksum(self, urldata): + return False + + def urldata_init(self, urldata, d): + if 'protocol' in urldata.parm and urldata.parm['protocol'] == 'git': + raise bb.fetch2.ParameterError( + "Invalid protocol - if you wish to fetch from a git " + + "repository using ssh, you need to use " + + "git:// prefix with protocol=ssh", urldata.url) + m = __pattern__.match(urldata.url) + path = m.group('path') + host = m.group('host') + urldata.localpath = os.path.join(d.getVar('DL_DIR', True), + os.path.basename(os.path.normpath(path))) + + def download(self, urldata, d): + dldir = d.getVar('DL_DIR', True) + + m = __pattern__.match(urldata.url) + path = m.group('path') + host = m.group('host') + port = m.group('port') + user = m.group('user') + password = m.group('pass') + + if port: + portarg = '-P %s' % port + else: + portarg = '' + + if user: + fr = user + if password: + fr += ':%s' % password + fr += '@%s' % host + else: + fr = host + fr += ':%s' % path + + + import commands + cmd = 'scp -B -r %s %s %s/' % ( + portarg, + commands.mkarg(fr), + commands.mkarg(dldir) + ) + + bb.fetch2.check_network_access(d, cmd, urldata.url) + + runfetchcmd(cmd, d) + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/svn.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/svn.py new file mode 100644 index 000000000..8a291935c --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/svn.py @@ -0,0 +1,197 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementation for svn. + +""" + +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2004 Marcin Juszkiewicz +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import os +import sys +import logging +import bb +import re +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import MissingParameterError +from bb.fetch2 import runfetchcmd +from bb.fetch2 import logger + +class Svn(FetchMethod): + """Class to fetch a module or modules from svn repositories""" + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with svn. + """ + return ud.type in ['svn'] + + def urldata_init(self, ud, d): + """ + init svn specific variable within url data + """ + if not "module" in ud.parm: + raise MissingParameterError('module', ud.url) + + ud.basecmd = d.getVar('FETCHCMD_svn', True) + + ud.module = ud.parm["module"] + + if not "path_spec" in ud.parm: + ud.path_spec = ud.module + else: + ud.path_spec = ud.parm["path_spec"] + + # Create paths to svn checkouts + relpath = self._strip_leading_slashes(ud.path) + ud.pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath) + ud.moddir = os.path.join(ud.pkgdir, ud.module) + + ud.setup_revisons(d) + + if 'rev' in ud.parm: + ud.revision = ud.parm['rev'] + + ud.localfile = data.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) + + def _buildsvncommand(self, ud, d, command): + """ + Build up an svn commandline based on ud + command is "fetch", "update", "info" + """ + + proto = ud.parm.get('protocol', 'svn') + + svn_rsh = None + if proto == "svn+ssh" and "rsh" in ud.parm: + svn_rsh = ud.parm["rsh"] + + svnroot = ud.host + ud.path + + options = [] + + options.append("--no-auth-cache") + + if ud.user: + options.append("--username %s" % ud.user) + + if ud.pswd: + options.append("--password %s" % ud.pswd) + + if command == "info": + svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) + elif command == "log1": + svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) + else: + suffix = "" + if ud.revision: + options.append("-r %s" % ud.revision) + suffix = "@%s" % (ud.revision) + + if command == "fetch": + transportuser = ud.parm.get("transportuser", "") + svncmd = "%s co %s %s://%s%s/%s%s %s" % (ud.basecmd, " ".join(options), proto, transportuser, svnroot, ud.module, suffix, ud.path_spec) + elif command == "update": + svncmd = "%s update %s" % (ud.basecmd, " ".join(options)) + else: + raise FetchError("Invalid svn command %s" % command, ud.url) + + if svn_rsh: + svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd) + + return svncmd + + def download(self, ud, d): + """Fetch url""" + + logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") + + if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): + svnupdatecmd = self._buildsvncommand(ud, d, "update") + logger.info("Update " + ud.url) + # update sources there + os.chdir(ud.moddir) + # We need to attempt to run svn upgrade first in case its an older working format + try: + runfetchcmd(ud.basecmd + " upgrade", d) + except FetchError: + pass + logger.debug(1, "Running %s", svnupdatecmd) + bb.fetch2.check_network_access(d, svnupdatecmd, ud.url) + runfetchcmd(svnupdatecmd, d) + else: + svnfetchcmd = self._buildsvncommand(ud, d, "fetch") + logger.info("Fetch " + ud.url) + # check out sources there + bb.utils.mkdirhier(ud.pkgdir) + os.chdir(ud.pkgdir) + logger.debug(1, "Running %s", svnfetchcmd) + bb.fetch2.check_network_access(d, svnfetchcmd, ud.url) + runfetchcmd(svnfetchcmd, d) + + scmdata = ud.parm.get("scmdata", "") + if scmdata == "keep": + tar_flags = "" + else: + tar_flags = "--exclude '.svn'" + + os.chdir(ud.pkgdir) + # tar them up to a defined filename + runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d, cleanup = [ud.localpath]) + + def clean(self, ud, d): + """ Clean SVN specific files and dirs """ + + bb.utils.remove(ud.localpath) + bb.utils.remove(ud.moddir, True) + + + def supports_srcrev(self): + return True + + def _revision_key(self, ud, d, name): + """ + Return a unique key for the url + """ + return "svn:" + ud.moddir + + def _latest_revision(self, ud, d, name): + """ + Return the latest upstream revision number + """ + bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "log1")) + + output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "log1"), d, True) + + # skip the first line, as per output of svn log + # then we expect the revision on the 2nd line + revision = re.search('^r([0-9]*)', output.splitlines()[1]).group(1) + + return revision + + def sortable_revision(self, ud, d, name): + """ + Return a sortable revision number which in our case is the revision number + """ + + return False, self._build_revision(ud, d) + + def _build_revision(self, ud, d): + return ud.revision diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py new file mode 100644 index 000000000..8bc9e93ca --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py @@ -0,0 +1,565 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +""" + +# Copyright (C) 2003, 2004 Chris Larson +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import re +import tempfile +import subprocess +import os +import logging +import bb +import urllib +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import logger +from bb.fetch2 import runfetchcmd +from bb.utils import export_proxies +from bs4 import BeautifulSoup +from bs4 import SoupStrainer + +class Wget(FetchMethod): + """Class to fetch urls via 'wget'""" + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with wget. + """ + return ud.type in ['http', 'https', 'ftp'] + + def recommends_checksum(self, urldata): + return True + + def urldata_init(self, ud, d): + if 'protocol' in ud.parm: + if ud.parm['protocol'] == 'git': + raise bb.fetch2.ParameterError("Invalid protocol - if you wish to fetch from a git repository using http, you need to instead use the git:// prefix with protocol=http", ud.url) + + if 'downloadfilename' in ud.parm: + ud.basename = ud.parm['downloadfilename'] + else: + ud.basename = os.path.basename(ud.path) + + ud.localfile = data.expand(urllib.unquote(ud.basename), d) + if not ud.localfile: + ud.localfile = data.expand(urllib.unquote(ud.host + ud.path).replace("/", "."), d) + + self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate" + + def _runwget(self, ud, d, command, quiet): + + logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) + bb.fetch2.check_network_access(d, command) + runfetchcmd(command, d, quiet) + + def download(self, ud, d): + """Fetch urls""" + + fetchcmd = self.basecmd + + if 'downloadfilename' in ud.parm: + dldir = d.getVar("DL_DIR", True) + bb.utils.mkdirhier(os.path.dirname(dldir + os.sep + ud.localfile)) + fetchcmd += " -O " + dldir + os.sep + ud.localfile + + uri = ud.url.split(";")[0] + if os.path.exists(ud.localpath): + # file exists, but we didnt complete it.. trying again.. + fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % uri) + else: + fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % uri) + + self._runwget(ud, d, fetchcmd, False) + + # Sanity check since wget can pretend it succeed when it didn't + # Also, this used to happen if sourceforge sent us to the mirror page + if not os.path.exists(ud.localpath): + raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri) + + if os.path.getsize(ud.localpath) == 0: + os.remove(ud.localpath) + raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri) + + return True + + def checkstatus(self, fetch, ud, d): + import urllib2, socket, httplib + from urllib import addinfourl + from bb.fetch2 import FetchConnectionCache + + class HTTPConnectionCache(httplib.HTTPConnection): + if fetch.connection_cache: + def connect(self): + """Connect to the host and port specified in __init__.""" + + sock = fetch.connection_cache.get_connection(self.host, self.port) + if sock: + self.sock = sock + else: + self.sock = socket.create_connection((self.host, self.port), + self.timeout, self.source_address) + fetch.connection_cache.add_connection(self.host, self.port, self.sock) + + if self._tunnel_host: + self._tunnel() + + class CacheHTTPHandler(urllib2.HTTPHandler): + def http_open(self, req): + return self.do_open(HTTPConnectionCache, req) + + def do_open(self, http_class, req): + """Return an addinfourl object for the request, using http_class. + + http_class must implement the HTTPConnection API from httplib. + The addinfourl return value is a file-like object. It also + has methods and attributes including: + - info(): return a mimetools.Message object for the headers + - geturl(): return the original request URL + - code: HTTP status code + """ + host = req.get_host() + if not host: + raise urlllib2.URLError('no host given') + + h = http_class(host, timeout=req.timeout) # will parse host:port + h.set_debuglevel(self._debuglevel) + + headers = dict(req.unredirected_hdrs) + headers.update(dict((k, v) for k, v in req.headers.items() + if k not in headers)) + + # We want to make an HTTP/1.1 request, but the addinfourl + # class isn't prepared to deal with a persistent connection. + # It will try to read all remaining data from the socket, + # which will block while the server waits for the next request. + # So make sure the connection gets closed after the (only) + # request. + + # Don't close connection when connection_cache is enabled, + if fetch.connection_cache is None: + headers["Connection"] = "close" + else: + headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0 + + headers = dict( + (name.title(), val) for name, val in headers.items()) + + if req._tunnel_host: + tunnel_headers = {} + proxy_auth_hdr = "Proxy-Authorization" + if proxy_auth_hdr in headers: + tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr] + # Proxy-Authorization should not be sent to origin + # server. + del headers[proxy_auth_hdr] + h.set_tunnel(req._tunnel_host, headers=tunnel_headers) + + try: + h.request(req.get_method(), req.get_selector(), req.data, headers) + except socket.error, err: # XXX what error? + # Don't close connection when cache is enabled. + if fetch.connection_cache is None: + h.close() + raise urllib2.URLError(err) + else: + try: + r = h.getresponse(buffering=True) + except TypeError: # buffering kw not supported + r = h.getresponse() + + # Pick apart the HTTPResponse object to get the addinfourl + # object initialized properly. + + # Wrap the HTTPResponse object in socket's file object adapter + # for Windows. That adapter calls recv(), so delegate recv() + # to read(). This weird wrapping allows the returned object to + # have readline() and readlines() methods. + + # XXX It might be better to extract the read buffering code + # out of socket._fileobject() and into a base class. + r.recv = r.read + + # no data, just have to read + r.read() + class fp_dummy(object): + def read(self): + return "" + def readline(self): + return "" + def close(self): + pass + + resp = addinfourl(fp_dummy(), r.msg, req.get_full_url()) + resp.code = r.status + resp.msg = r.reason + + # Close connection when server request it. + if fetch.connection_cache is not None: + if 'Connection' in r.msg and r.msg['Connection'] == 'close': + fetch.connection_cache.remove_connection(h.host, h.port) + + return resp + + class HTTPMethodFallback(urllib2.BaseHandler): + """ + Fallback to GET if HEAD is not allowed (405 HTTP error) + """ + def http_error_405(self, req, fp, code, msg, headers): + fp.read() + fp.close() + + newheaders = dict((k,v) for k,v in req.headers.items() + if k.lower() not in ("content-length", "content-type")) + return self.parent.open(urllib2.Request(req.get_full_url(), + headers=newheaders, + origin_req_host=req.get_origin_req_host(), + unverifiable=True)) + + """ + Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 + Forbidden when they actually mean 405 Method Not Allowed. + """ + http_error_403 = http_error_405 + + """ + Some servers (e.g. FusionForge) returns 406 Not Acceptable when they + actually mean 405 Method Not Allowed. + """ + http_error_406 = http_error_405 + + class FixedHTTPRedirectHandler(urllib2.HTTPRedirectHandler): + """ + urllib2.HTTPRedirectHandler resets the method to GET on redirect, + when we want to follow redirects using the original method. + """ + def redirect_request(self, req, fp, code, msg, headers, newurl): + newreq = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) + newreq.get_method = lambda: req.get_method() + return newreq + exported_proxies = export_proxies(d) + + handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] + if export_proxies: + handlers.append(urllib2.ProxyHandler()) + handlers.append(CacheHTTPHandler()) + # XXX: Since Python 2.7.9 ssl cert validation is enabled by default + # see PEP-0476, this causes verification errors on some https servers + # so disable by default. + import ssl + if hasattr(ssl, '_create_unverified_context'): + handlers.append(urllib2.HTTPSHandler(context=ssl._create_unverified_context())) + opener = urllib2.build_opener(*handlers) + + try: + uri = ud.url.split(";")[0] + r = urllib2.Request(uri) + r.get_method = lambda: "HEAD" + opener.open(r) + except urllib2.URLError as e: + # debug for now to avoid spamming the logs in e.g. remote sstate searches + logger.debug(2, "checkstatus() urlopen failed: %s" % e) + return False + return True + + def _parse_path(self, regex, s): + """ + Find and group name, version and archive type in the given string s + """ + + m = regex.search(s) + if m: + pname = '' + pver = '' + ptype = '' + + mdict = m.groupdict() + if 'name' in mdict.keys(): + pname = mdict['name'] + if 'pver' in mdict.keys(): + pver = mdict['pver'] + if 'type' in mdict.keys(): + ptype = mdict['type'] + + bb.debug(3, "_parse_path: %s, %s, %s" % (pname, pver, ptype)) + + return (pname, pver, ptype) + + return None + + def _modelate_version(self, version): + if version[0] in ['.', '-']: + if version[1].isdigit(): + version = version[1] + version[0] + version[2:len(version)] + else: + version = version[1:len(version)] + + version = re.sub('-', '.', version) + version = re.sub('_', '.', version) + version = re.sub('(rc)+', '.1000.', version) + version = re.sub('(beta)+', '.100.', version) + version = re.sub('(alpha)+', '.10.', version) + if version[0] == 'v': + version = version[1:len(version)] + return version + + def _vercmp(self, old, new): + """ + Check whether 'new' is newer than 'old' version. We use existing vercmp() for the + purpose. PE is cleared in comparison as it's not for build, and PR is cleared too + for simplicity as it's somehow difficult to get from various upstream format + """ + + (oldpn, oldpv, oldsuffix) = old + (newpn, newpv, newsuffix) = new + + """ + Check for a new suffix type that we have never heard of before + """ + if (newsuffix): + m = self.suffix_regex_comp.search(newsuffix) + if not m: + bb.warn("%s has a possible unknown suffix: %s" % (newpn, newsuffix)) + return False + + """ + Not our package so ignore it + """ + if oldpn != newpn: + return False + + oldpv = self._modelate_version(oldpv) + newpv = self._modelate_version(newpv) + + return bb.utils.vercmp(("0", oldpv, ""), ("0", newpv, "")) + + def _fetch_index(self, uri, ud, d): + """ + Run fetch checkstatus to get directory information + """ + f = tempfile.NamedTemporaryFile() + + agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12" + fetchcmd = self.basecmd + fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'" + try: + self._runwget(ud, d, fetchcmd, True) + fetchresult = f.read() + except bb.fetch2.BBFetchException: + fetchresult = "" + + f.close() + return fetchresult + + def _check_latest_version(self, url, package, package_regex, current_version, ud, d): + """ + Return the latest version of a package inside a given directory path + If error or no version, return "" + """ + valid = 0 + version = ['', '', ''] + + bb.debug(3, "VersionURL: %s" % (url)) + soup = BeautifulSoup(self._fetch_index(url, ud, d), "html.parser", parse_only=SoupStrainer("a")) + if not soup: + bb.debug(3, "*** %s NO SOUP" % (url)) + return "" + + for line in soup.find_all('a', href=True): + bb.debug(3, "line['href'] = '%s'" % (line['href'])) + bb.debug(3, "line = '%s'" % (str(line))) + + newver = self._parse_path(package_regex, line['href']) + if not newver: + newver = self._parse_path(package_regex, str(line)) + + if newver: + bb.debug(3, "Upstream version found: %s" % newver[1]) + if valid == 0: + version = newver + valid = 1 + elif self._vercmp(version, newver) < 0: + version = newver + + pupver = re.sub('_', '.', version[1]) + + bb.debug(3, "*** %s -> UpstreamVersion = %s (CurrentVersion = %s)" % + (package, pupver or "N/A", current_version[1])) + + if valid: + return pupver + + return "" + + def _check_latest_version_by_dir(self, dirver, package, package_regex, + current_version, ud, d): + """ + Scan every directory in order to get upstream version. + """ + version_dir = ['', '', ''] + version = ['', '', ''] + + dirver_regex = re.compile("(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") + s = dirver_regex.search(dirver) + if s: + version_dir[1] = s.group('ver') + else: + version_dir[1] = dirver + + dirs_uri = bb.fetch.encodeurl([ud.type, ud.host, + ud.path.split(dirver)[0], ud.user, ud.pswd, {}]) + bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package)) + + soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d), "html.parser", parse_only=SoupStrainer("a")) + if not soup: + return version[1] + + for line in soup.find_all('a', href=True): + s = dirver_regex.search(line['href'].strip("/")) + if s: + sver = s.group('ver') + + # When prefix is part of the version directory it need to + # ensure that only version directory is used so remove previous + # directories if exists. + # + # Example: pfx = '/dir1/dir2/v' and version = '2.5' the expected + # result is v2.5. + spfx = s.group('pfx').split('/')[-1] + + version_dir_new = ['', sver, ''] + if self._vercmp(version_dir, version_dir_new) <= 0: + dirver_new = spfx + sver + path = ud.path.replace(dirver, dirver_new, True) \ + .split(package)[0] + uri = bb.fetch.encodeurl([ud.type, ud.host, path, + ud.user, ud.pswd, {}]) + + pupver = self._check_latest_version(uri, + package, package_regex, current_version, ud, d) + if pupver: + version[1] = pupver + + version_dir = version_dir_new + + return version[1] + + def _init_regexes(self, package, ud, d): + """ + Match as many patterns as possible such as: + gnome-common-2.20.0.tar.gz (most common format) + gtk+-2.90.1.tar.gz + xf86-input-synaptics-12.6.9.tar.gz + dri2proto-2.3.tar.gz + blktool_4.orig.tar.gz + libid3tag-0.15.1b.tar.gz + unzip552.tar.gz + icu4c-3_6-src.tgz + genext2fs_1.3.orig.tar.gz + gst-fluendo-mp3 + """ + # match most patterns which uses "-" as separator to version digits + pn_prefix1 = "[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]" + # a loose pattern such as for unzip552.tar.gz + pn_prefix2 = "[a-zA-Z]+" + # a loose pattern such as for 80325-quicky-0.4.tar.gz + pn_prefix3 = "[0-9]+[-]?[a-zA-Z]+" + # Save the Package Name (pn) Regex for use later + pn_regex = "(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3) + + # match version + pver_regex = "(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)" + + # match arch + parch_regex = "-source|_all_" + + # src.rpm extension was added only for rpm package. Can be removed if the rpm + # packaged will always be considered as having to be manually upgraded + psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" + + # match name, version and archive type of a package + package_regex_comp = re.compile("(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" + % (pn_regex, pver_regex, parch_regex, psuffix_regex)) + self.suffix_regex_comp = re.compile(psuffix_regex) + + # compile regex, can be specific by package or generic regex + pn_regex = d.getVar('UPSTREAM_CHECK_REGEX', True) + if pn_regex: + package_custom_regex_comp = re.compile(pn_regex) + else: + version = self._parse_path(package_regex_comp, package) + if version: + package_custom_regex_comp = re.compile( + "(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" % + (re.escape(version[0]), pver_regex, parch_regex, psuffix_regex)) + else: + package_custom_regex_comp = None + + return package_custom_regex_comp + + def latest_versionstring(self, ud, d): + """ + Manipulate the URL and try to obtain the latest package version + + sanity check to ensure same name and type. + """ + package = ud.path.split("/")[-1] + current_version = ['', d.getVar('PV', True), ''] + + """possible to have no version in pkg name, such as spectrum-fw""" + if not re.search("\d+", package): + current_version[1] = re.sub('_', '.', current_version[1]) + current_version[1] = re.sub('-', '.', current_version[1]) + return (current_version[1], '') + + package_regex = self._init_regexes(package, ud, d) + if package_regex is None: + bb.warn("latest_versionstring: package %s don't match pattern" % (package)) + return ('', '') + bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern)) + + uri = "" + regex_uri = d.getVar("UPSTREAM_CHECK_URI", True) + if not regex_uri: + path = ud.path.split(package)[0] + + # search for version matches on folders inside the path, like: + # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz + dirver_regex = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") + m = dirver_regex.search(path) + if m: + pn = d.getVar('PN', True) + dirver = m.group('dirver') + + dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn))) + if not dirver_pn_regex.search(dirver): + return (self._check_latest_version_by_dir(dirver, + package, package_regex, current_version, ud, d), '') + + uri = bb.fetch.encodeurl([ud.type, ud.host, path, ud.user, ud.pswd, {}]) + else: + uri = regex_uri + + return (self._check_latest_version(uri, package, package_regex, + current_version, ud, d), '') diff --git a/import-layers/yocto-poky/bitbake/lib/bb/main.py b/import-layers/yocto-poky/bitbake/lib/bb/main.py new file mode 100755 index 000000000..e30217369 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/main.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH +# Copyright (C) 2006 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys +import logging +import optparse +import warnings + +import bb +from bb import event +import bb.msg +from bb import cooker +from bb import ui +from bb import server +from bb import cookerdata + +logger = logging.getLogger("BitBake") + +class BBMainException(Exception): + pass + +def present_options(optionlist): + if len(optionlist) > 1: + return ' or '.join([', '.join(optionlist[:-1]), optionlist[-1]]) + else: + return optionlist[0] + +class BitbakeHelpFormatter(optparse.IndentedHelpFormatter): + def format_option(self, option): + # We need to do this here rather than in the text we supply to + # add_option() because we don't want to call list_extension_modules() + # on every execution (since it imports all of the modules) + # Note also that we modify option.help rather than the returned text + # - this is so that we don't have to re-format the text ourselves + if option.dest == 'ui': + valid_uis = list_extension_modules(bb.ui, 'main') + option.help = option.help.replace('@CHOICES@', present_options(valid_uis)) + elif option.dest == 'servertype': + valid_server_types = list_extension_modules(bb.server, 'BitBakeServer') + option.help = option.help.replace('@CHOICES@', present_options(valid_server_types)) + + return optparse.IndentedHelpFormatter.format_option(self, option) + +def list_extension_modules(pkg, checkattr): + """ + Lists extension modules in a specific Python package + (e.g. UIs, servers). NOTE: Calling this function will import all of the + submodules of the specified module in order to check for the specified + attribute; this can have unusual side-effects. As a result, this should + only be called when displaying help text or error messages. + Parameters: + pkg: previously imported Python package to list + checkattr: attribute to look for in module to determine if it's valid + as the type of extension you are looking for + """ + import pkgutil + pkgdir = os.path.dirname(pkg.__file__) + + modules = [] + for _, modulename, _ in pkgutil.iter_modules([pkgdir]): + if os.path.isdir(os.path.join(pkgdir, modulename)): + # ignore directories + continue + try: + module = __import__(pkg.__name__, fromlist=[modulename]) + except: + # If we can't import it, it's not valid + continue + module_if = getattr(module, modulename) + if getattr(module_if, 'hidden_extension', False): + continue + if not checkattr or hasattr(module_if, checkattr): + modules.append(modulename) + return modules + +def import_extension_module(pkg, modulename, checkattr): + try: + # Dynamically load the UI based on the ui name. Although we + # suggest a fixed set this allows you to have flexibility in which + # ones are available. + module = __import__(pkg.__name__, fromlist = [modulename]) + return getattr(module, modulename) + except AttributeError: + raise BBMainException('FATAL: Unable to import extension module "%s" from %s. Valid extension modules: %s' % (modulename, pkg.__name__, present_options(list_extension_modules(pkg, checkattr)))) + + +# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others""" +warnlog = logging.getLogger("BitBake.Warnings") +_warnings_showwarning = warnings.showwarning +def _showwarning(message, category, filename, lineno, file=None, line=None): + if file is not None: + if _warnings_showwarning is not None: + _warnings_showwarning(message, category, filename, lineno, file, line) + else: + s = warnings.formatwarning(message, category, filename, lineno) + warnlog.warn(s) + +warnings.showwarning = _showwarning +warnings.filterwarnings("ignore") +warnings.filterwarnings("default", module="(<string>$|(oe|bb)\.)") +warnings.filterwarnings("ignore", category=PendingDeprecationWarning) +warnings.filterwarnings("ignore", category=ImportWarning) +warnings.filterwarnings("ignore", category=DeprecationWarning, module="<string>$") +warnings.filterwarnings("ignore", message="With-statements now directly support multiple context managers") + +class BitBakeConfigParameters(cookerdata.ConfigParameters): + + def parseCommandLine(self, argv=sys.argv): + parser = optparse.OptionParser( + formatter = BitbakeHelpFormatter(), + version = "BitBake Build Tool Core version %s" % bb.__version__, + usage = """%prog [options] [recipename/target recipe:do_task ...] + + Executes the specified task (default is 'build') for a given set of target recipes (.bb files). + It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which + will provide the layer, BBFILES and other configuration information.""") + + parser.add_option("-b", "--buildfile", help = "Execute tasks from a specific .bb recipe directly. WARNING: Does not handle any dependencies from other recipes.", + action = "store", dest = "buildfile", default = None) + + parser.add_option("-k", "--continue", help = "Continue as much as possible after an error. While the target that failed and anything depending on it cannot be built, as much as possible will be built before stopping.", + action = "store_false", dest = "abort", default = True) + + parser.add_option("-a", "--tryaltconfigs", help = "Continue with builds by trying to use alternative providers where possible.", + action = "store_true", dest = "tryaltconfigs", default = False) + + parser.add_option("-f", "--force", help = "Force the specified targets/task to run (invalidating any existing stamp file).", + action = "store_true", dest = "force", default = False) + + parser.add_option("-c", "--cmd", help = "Specify the task to execute. The exact options available depend on the metadata. Some examples might be 'compile' or 'populate_sysroot' or 'listtasks' may give a list of the tasks available.", + action = "store", dest = "cmd") + + parser.add_option("-C", "--clear-stamp", help = "Invalidate the stamp for the specified task such as 'compile' and then run the default task for the specified target(s).", + action = "store", dest = "invalidate_stamp") + + parser.add_option("-r", "--read", help = "Read the specified file before bitbake.conf.", + action = "append", dest = "prefile", default = []) + + parser.add_option("-R", "--postread", help = "Read the specified file after bitbake.conf.", + action = "append", dest = "postfile", default = []) + + parser.add_option("-v", "--verbose", help = "Output more log message data to the terminal.", + action = "store_true", dest = "verbose", default = False) + + parser.add_option("-D", "--debug", help = "Increase the debug level. You can specify this more than once.", + action = "count", dest="debug", default = 0) + + parser.add_option("-n", "--dry-run", help = "Don't execute, just go through the motions.", + action = "store_true", dest = "dry_run", default = False) + + parser.add_option("-S", "--dump-signatures", help = "Dump out the signature construction information, with no task execution. The SIGNATURE_HANDLER parameter is passed to the handler. Two common values are none and printdiff but the handler may define more/less. none means only dump the signature, printdiff means compare the dumped signature with the cached one.", + action = "append", dest = "dump_signatures", default = [], metavar="SIGNATURE_HANDLER") + + parser.add_option("-p", "--parse-only", help = "Quit after parsing the BB recipes.", + action = "store_true", dest = "parse_only", default = False) + + parser.add_option("-s", "--show-versions", help = "Show current and preferred versions of all recipes.", + action = "store_true", dest = "show_versions", default = False) + + parser.add_option("-e", "--environment", help = "Show the global or per-recipe environment complete with information about where variables were set/changed.", + action = "store_true", dest = "show_environment", default = False) + + parser.add_option("-g", "--graphviz", help = "Save dependency tree information for the specified targets in the dot syntax.", + action = "store_true", dest = "dot_graph", default = False) + + parser.add_option("-I", "--ignore-deps", help = """Assume these dependencies don't exist and are already provided (equivalent to ASSUME_PROVIDED). Useful to make dependency graphs more appealing""", + action = "append", dest = "extra_assume_provided", default = []) + + parser.add_option("-l", "--log-domains", help = """Show debug logging for the specified logging domains""", + action = "append", dest = "debug_domains", default = []) + + parser.add_option("-P", "--profile", help = "Profile the command and save reports.", + action = "store_true", dest = "profile", default = False) + + env_ui = os.environ.get('BITBAKE_UI', None) + default_ui = env_ui or 'knotty' + # @CHOICES@ is substituted out by BitbakeHelpFormatter above + parser.add_option("-u", "--ui", help = "The user interface to use (@CHOICES@ - default %default).", + action="store", dest="ui", default=default_ui) + + # @CHOICES@ is substituted out by BitbakeHelpFormatter above + parser.add_option("-t", "--servertype", help = "Choose which server type to use (@CHOICES@ - default %default).", + action = "store", dest = "servertype", default = "process") + + parser.add_option("", "--token", help = "Specify the connection token to be used when connecting to a remote server.", + action = "store", dest = "xmlrpctoken") + + parser.add_option("", "--revisions-changed", help = "Set the exit code depending on whether upstream floating revisions have changed or not.", + action = "store_true", dest = "revisions_changed", default = False) + + parser.add_option("", "--server-only", help = "Run bitbake without a UI, only starting a server (cooker) process.", + action = "store_true", dest = "server_only", default = False) + + parser.add_option("-B", "--bind", help = "The name/address for the bitbake server to bind to.", + action = "store", dest = "bind", default = False) + + parser.add_option("", "--no-setscene", help = "Do not run any setscene tasks. sstate will be ignored and everything needed, built.", + action = "store_true", dest = "nosetscene", default = False) + + parser.add_option("", "--setscene-only", help = "Only run setscene tasks, don't run any real tasks.", + action = "store_true", dest = "setsceneonly", default = False) + + parser.add_option("", "--remote-server", help = "Connect to the specified server.", + action = "store", dest = "remote_server", default = False) + + parser.add_option("-m", "--kill-server", help = "Terminate the remote server.", + action = "store_true", dest = "kill_server", default = False) + + parser.add_option("", "--observe-only", help = "Connect to a server as an observing-only client.", + action = "store_true", dest = "observe_only", default = False) + + parser.add_option("", "--status-only", help = "Check the status of the remote bitbake server.", + action = "store_true", dest = "status_only", default = False) + + parser.add_option("-w", "--write-log", help = "Writes the event log of the build to a bitbake event json file. Use '' (empty string) to assign the name automatically.", + action = "store", dest = "writeeventlog") + + options, targets = parser.parse_args(argv) + + # some environmental variables set also configuration options + if "BBSERVER" in os.environ: + options.servertype = "xmlrpc" + options.remote_server = os.environ["BBSERVER"] + + if "BBTOKEN" in os.environ: + options.xmlrpctoken = os.environ["BBTOKEN"] + + if "BBEVENTLOG" in os.environ: + options.writeeventlog = os.environ["BBEVENTLOG"] + + # fill in proper log name if not supplied + if options.writeeventlog is not None and len(options.writeeventlog) == 0: + import datetime + options.writeeventlog = "bitbake_eventlog_%s.json" % datetime.datetime.now().strftime("%Y%m%d%H%M%S") + + # if BBSERVER says to autodetect, let's do that + if options.remote_server: + [host, port] = options.remote_server.split(":", 2) + port = int(port) + # use automatic port if port set to -1, means read it from + # the bitbake.lock file; this is a bit tricky, but we always expect + # to be in the base of the build directory if we need to have a + # chance to start the server later, anyway + if port == -1: + lock_location = "./bitbake.lock" + # we try to read the address at all times; if the server is not started, + # we'll try to start it after the first connect fails, below + try: + lf = open(lock_location, 'r') + remotedef = lf.readline() + [host, port] = remotedef.split(":") + port = int(port) + lf.close() + options.remote_server = remotedef + except Exception as e: + raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e)) + + return options, targets[1:] + + +def start_server(servermodule, configParams, configuration, features): + server = servermodule.BitBakeServer() + single_use = not configParams.server_only + if configParams.bind: + (host, port) = configParams.bind.split(':') + server.initServer((host, int(port)), single_use) + configuration.interface = [ server.serverImpl.host, server.serverImpl.port ] + else: + server.initServer(single_use=single_use) + configuration.interface = [] + + try: + configuration.setServerRegIdleCallback(server.getServerIdleCB()) + + cooker = bb.cooker.BBCooker(configuration, features) + + server.addcooker(cooker) + server.saveConnectionDetails() + except Exception as e: + exc_info = sys.exc_info() + while hasattr(server, "event_queue"): + try: + import queue + except ImportError: + import Queue as queue + try: + event = server.event_queue.get(block=False) + except (queue.Empty, IOError): + break + if isinstance(event, logging.LogRecord): + logger.handle(event) + raise exc_info[1], None, exc_info[2] + server.detach() + cooker.lock.close() + return server + + +def bitbake_main(configParams, configuration): + + # Python multiprocessing requires /dev/shm on Linux + if sys.platform.startswith('linux') and not os.access('/dev/shm', os.W_OK | os.X_OK): + raise BBMainException("FATAL: /dev/shm does not exist or is not writable") + + # Unbuffer stdout to avoid log truncation in the event + # of an unorderly exit as well as to provide timely + # updates to log files for use with tail + try: + if sys.stdout.name == '<stdout>': + sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) + except: + pass + + + configuration.setConfigParameters(configParams) + + ui_module = import_extension_module(bb.ui, configParams.ui, 'main') + servermodule = import_extension_module(bb.server, configParams.servertype, 'BitBakeServer') + + if configParams.server_only: + if configParams.servertype != "xmlrpc": + raise BBMainException("FATAL: If '--server-only' is defined, we must set the " + "servertype as 'xmlrpc'.\n") + if not configParams.bind: + raise BBMainException("FATAL: The '--server-only' option requires a name/address " + "to bind to with the -B option.\n") + if configParams.remote_server: + raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" % + ("the BBSERVER environment variable" if "BBSERVER" in os.environ \ + else "the '--remote-server' option" )) + + if configParams.bind and configParams.servertype != "xmlrpc": + raise BBMainException("FATAL: If '-B' or '--bind' is defined, we must " + "set the servertype as 'xmlrpc'.\n") + + if configParams.remote_server and configParams.servertype != "xmlrpc": + raise BBMainException("FATAL: If '--remote-server' is defined, we must " + "set the servertype as 'xmlrpc'.\n") + + if configParams.observe_only and (not configParams.remote_server or configParams.bind): + raise BBMainException("FATAL: '--observe-only' can only be used by UI clients " + "connecting to a server.\n") + + if configParams.kill_server and not configParams.remote_server: + raise BBMainException("FATAL: '--kill-server' can only be used to terminate a remote server") + + if "BBDEBUG" in os.environ: + level = int(os.environ["BBDEBUG"]) + if level > configuration.debug: + configuration.debug = level + + bb.msg.init_msgconfig(configParams.verbose, configuration.debug, + configuration.debug_domains) + + # Ensure logging messages get sent to the UI as events + handler = bb.event.LogHandler() + if not configParams.status_only: + # In status only mode there are no logs and no UI + logger.addHandler(handler) + + # Clear away any spurious environment variables while we stoke up the cooker + cleanedvars = bb.utils.clean_environment() + + featureset = [] + if not configParams.server_only: + # Collect the feature set for the UI + featureset = getattr(ui_module, "featureSet", []) + + if configParams.server_only: + for param in ('prefile', 'postfile'): + value = getattr(configParams, param) + if value: + setattr(configuration, "%s_server" % param, value) + param = "%s_server" % param + + if not configParams.remote_server: + # we start a server with a given configuration + server = start_server(servermodule, configParams, configuration, featureset) + bb.event.ui_queue = [] + else: + # we start a stub server that is actually a XMLRPClient that connects to a real server + server = servermodule.BitBakeXMLRPCClient(configParams.observe_only, configParams.xmlrpctoken) + server.saveConnectionDetails(configParams.remote_server) + + + if not configParams.server_only: + try: + server_connection = server.establishConnection(featureset) + except Exception as e: + bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e))) + + if configParams.kill_server: + server_connection.connection.terminateServer() + bb.event.ui_queue = [] + return 0 + + server_connection.setupEventQueue() + + # Restore the environment in case the UI needs it + for k in cleanedvars: + os.environ[k] = cleanedvars[k] + + logger.removeHandler(handler) + + + if configParams.status_only: + server_connection.terminate() + return 0 + + try: + return ui_module.main(server_connection.connection, server_connection.events, configParams) + finally: + bb.event.ui_queue = [] + server_connection.terminate() + else: + print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host, server.serverImpl.port)) + return 0 + + return 1 diff --git a/import-layers/yocto-poky/bitbake/lib/bb/methodpool.py b/import-layers/yocto-poky/bitbake/lib/bb/methodpool.py new file mode 100644 index 000000000..49aed3338 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/methodpool.py @@ -0,0 +1,40 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# +# Copyright (C) 2006 Holger Hans Peter Freyther +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from bb.utils import better_compile, better_exec + +def insert_method(modulename, code, fn, lineno): + """ + Add code of a module should be added. The methods + will be simply added, no checking will be done + """ + comp = better_compile(code, modulename, fn, lineno=lineno) + better_exec(comp, None, code, fn) + +compilecache = {} + +def compile_cache(code): + h = hash(code) + if h in compilecache: + return compilecache[h] + return None + +def compile_cache_add(code, compileobj): + h = hash(code) + compilecache[h] = compileobj diff --git a/import-layers/yocto-poky/bitbake/lib/bb/monitordisk.py b/import-layers/yocto-poky/bitbake/lib/bb/monitordisk.py new file mode 100644 index 000000000..466523c6e --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/monitordisk.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2012 Robert Yang +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os, logging, re, sys +import bb +logger = logging.getLogger("BitBake.Monitor") + +def printErr(info): + logger.error("%s\n Disk space monitor will NOT be enabled" % info) + +def convertGMK(unit): + + """ Convert the space unit G, M, K, the unit is case-insensitive """ + + unitG = re.match('([1-9][0-9]*)[gG]\s?$', unit) + if unitG: + return int(unitG.group(1)) * (1024 ** 3) + unitM = re.match('([1-9][0-9]*)[mM]\s?$', unit) + if unitM: + return int(unitM.group(1)) * (1024 ** 2) + unitK = re.match('([1-9][0-9]*)[kK]\s?$', unit) + if unitK: + return int(unitK.group(1)) * 1024 + unitN = re.match('([1-9][0-9]*)\s?$', unit) + if unitN: + return int(unitN.group(1)) + else: + return None + +def getMountedDev(path): + + """ Get the device mounted at the path, uses /proc/mounts """ + + # Get the mount point of the filesystem containing path + # st_dev is the ID of device containing file + parentDev = os.stat(path).st_dev + currentDev = parentDev + # When the current directory's device is different from the + # parent's, then the current directory is a mount point + while parentDev == currentDev: + mountPoint = path + # Use dirname to get the parent's directory + path = os.path.dirname(path) + # Reach the "/" + if path == mountPoint: + break + parentDev= os.stat(path).st_dev + + try: + with open("/proc/mounts", "r") as ifp: + for line in ifp: + procLines = line.rstrip('\n').split() + if procLines[1] == mountPoint: + return procLines[0] + except EnvironmentError: + pass + return None + +def getDiskData(BBDirs, configuration): + + """Prepare disk data for disk space monitor""" + + # Save the device IDs, need the ID to be unique (the dictionary's key is + # unique), so that when more than one directory is located on the same + # device, we just monitor it once + devDict = {} + for pathSpaceInode in BBDirs.split(): + # The input format is: "dir,space,inode", dir is a must, space + # and inode are optional + pathSpaceInodeRe = re.match('([^,]*),([^,]*),([^,]*),?(.*)', pathSpaceInode) + if not pathSpaceInodeRe: + printErr("Invalid value in BB_DISKMON_DIRS: %s" % pathSpaceInode) + return None + + action = pathSpaceInodeRe.group(1) + if action not in ("ABORT", "STOPTASKS", "WARN"): + printErr("Unknown disk space monitor action: %s" % action) + return None + + path = os.path.realpath(pathSpaceInodeRe.group(2)) + if not path: + printErr("Invalid path value in BB_DISKMON_DIRS: %s" % pathSpaceInode) + return None + + # The disk space or inode is optional, but it should have a correct + # value once it is specified + minSpace = pathSpaceInodeRe.group(3) + if minSpace: + minSpace = convertGMK(minSpace) + if not minSpace: + printErr("Invalid disk space value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(3)) + return None + else: + # None means that it is not specified + minSpace = None + + minInode = pathSpaceInodeRe.group(4) + if minInode: + minInode = convertGMK(minInode) + if not minInode: + printErr("Invalid inode value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(4)) + return None + else: + # None means that it is not specified + minInode = None + + if minSpace is None and minInode is None: + printErr("No disk space or inode value in found BB_DISKMON_DIRS: %s" % pathSpaceInode) + return None + # mkdir for the directory since it may not exist, for example the + # DL_DIR may not exist at the very beginning + if not os.path.exists(path): + bb.utils.mkdirhier(path) + dev = getMountedDev(path) + # Use path/action as the key + devDict[os.path.join(path, action)] = [dev, minSpace, minInode] + + return devDict + +def getInterval(configuration): + + """ Get the disk space interval """ + + # The default value is 50M and 5K. + spaceDefault = 50 * 1024 * 1024 + inodeDefault = 5 * 1024 + + interval = configuration.getVar("BB_DISKMON_WARNINTERVAL", True) + if not interval: + return spaceDefault, inodeDefault + else: + # The disk space or inode interval is optional, but it should + # have a correct value once it is specified + intervalRe = re.match('([^,]*),?\s*(.*)', interval) + if intervalRe: + intervalSpace = intervalRe.group(1) + if intervalSpace: + intervalSpace = convertGMK(intervalSpace) + if not intervalSpace: + printErr("Invalid disk space interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(1)) + return None, None + else: + intervalSpace = spaceDefault + intervalInode = intervalRe.group(2) + if intervalInode: + intervalInode = convertGMK(intervalInode) + if not intervalInode: + printErr("Invalid disk inode interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(2)) + return None, None + else: + intervalInode = inodeDefault + return intervalSpace, intervalInode + else: + printErr("Invalid interval value in BB_DISKMON_WARNINTERVAL: %s" % interval) + return None, None + +class diskMonitor: + + """Prepare the disk space monitor data""" + + def __init__(self, configuration): + + self.enableMonitor = False + self.configuration = configuration + + BBDirs = configuration.getVar("BB_DISKMON_DIRS", True) or None + if BBDirs: + self.devDict = getDiskData(BBDirs, configuration) + if self.devDict: + self.spaceInterval, self.inodeInterval = getInterval(configuration) + if self.spaceInterval and self.inodeInterval: + self.enableMonitor = True + # These are for saving the previous disk free space and inode, we + # use them to avoid printing too many warning messages + self.preFreeS = {} + self.preFreeI = {} + # This is for STOPTASKS and ABORT, to avoid printing the message + # repeatedly while waiting for the tasks to finish + self.checked = {} + for k in self.devDict: + self.preFreeS[k] = 0 + self.preFreeI[k] = 0 + self.checked[k] = False + if self.spaceInterval is None and self.inodeInterval is None: + self.enableMonitor = False + + def check(self, rq): + + """ Take action for the monitor """ + + if self.enableMonitor: + for k in self.devDict: + path = os.path.dirname(k) + action = os.path.basename(k) + dev = self.devDict[k][0] + minSpace = self.devDict[k][1] + minInode = self.devDict[k][2] + + st = os.statvfs(path) + + # The free space, float point number + freeSpace = st.f_bavail * st.f_frsize + + if minSpace and freeSpace < minSpace: + # Always show warning, the self.checked would always be False if the action is WARN + if self.preFreeS[k] == 0 or self.preFreeS[k] - freeSpace > self.spaceInterval and not self.checked[k]: + logger.warn("The free space of %s (%s) is running low (%.3fGB left)" % \ + (path, dev, freeSpace / 1024 / 1024 / 1024.0)) + self.preFreeS[k] = freeSpace + + if action == "STOPTASKS" and not self.checked[k]: + logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!") + self.checked[k] = True + rq.finish_runqueue(False) + bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration) + elif action == "ABORT" and not self.checked[k]: + logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!") + self.checked[k] = True + rq.finish_runqueue(True) + bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration) + + # The free inodes, float point number + freeInode = st.f_favail + + if minInode and freeInode < minInode: + # Some filesystems use dynamic inodes so can't run out + # (e.g. btrfs). This is reported by the inode count being 0. + if st.f_files == 0: + self.devDict[k][2] = None + continue + # Always show warning, the self.checked would always be False if the action is WARN + if self.preFreeI[k] == 0 or self.preFreeI[k] - freeInode > self.inodeInterval and not self.checked[k]: + logger.warn("The free inode of %s (%s) is running low (%.3fK left)" % \ + (path, dev, freeInode / 1024.0)) + self.preFreeI[k] = freeInode + + if action == "STOPTASKS" and not self.checked[k]: + logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!") + self.checked[k] = True + rq.finish_runqueue(False) + bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration) + elif action == "ABORT" and not self.checked[k]: + logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!") + self.checked[k] = True + rq.finish_runqueue(True) + bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration) + return diff --git a/import-layers/yocto-poky/bitbake/lib/bb/msg.py b/import-layers/yocto-poky/bitbake/lib/bb/msg.py new file mode 100644 index 000000000..786b5aef4 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/msg.py @@ -0,0 +1,199 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'msg' implementation + +Message handling infrastructure for bitbake + +""" + +# Copyright (C) 2006 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import copy +import logging +import collections +from itertools import groupby +import warnings +import bb +import bb.event + +class BBLogFormatter(logging.Formatter): + """Formatter which ensures that our 'plain' messages (logging.INFO + 1) are used as is""" + + DEBUG3 = logging.DEBUG - 2 + DEBUG2 = logging.DEBUG - 1 + DEBUG = logging.DEBUG + VERBOSE = logging.INFO - 1 + NOTE = logging.INFO + PLAIN = logging.INFO + 1 + ERROR = logging.ERROR + WARNING = logging.WARNING + CRITICAL = logging.CRITICAL + + levelnames = { + DEBUG3 : 'DEBUG', + DEBUG2 : 'DEBUG', + DEBUG : 'DEBUG', + VERBOSE: 'NOTE', + NOTE : 'NOTE', + PLAIN : '', + WARNING : 'WARNING', + ERROR : 'ERROR', + CRITICAL: 'ERROR', + } + + color_enabled = False + BASECOLOR, BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(29,38) + + COLORS = { + DEBUG3 : CYAN, + DEBUG2 : CYAN, + DEBUG : CYAN, + VERBOSE : BASECOLOR, + NOTE : BASECOLOR, + PLAIN : BASECOLOR, + WARNING : YELLOW, + ERROR : RED, + CRITICAL: RED, + } + + BLD = '\033[1;%dm' + STD = '\033[%dm' + RST = '\033[0m' + + def getLevelName(self, levelno): + try: + return self.levelnames[levelno] + except KeyError: + self.levelnames[levelno] = value = 'Level %d' % levelno + return value + + def format(self, record): + record.levelname = self.getLevelName(record.levelno) + if record.levelno == self.PLAIN: + msg = record.getMessage() + else: + if self.color_enabled: + record = self.colorize(record) + msg = logging.Formatter.format(self, record) + + if hasattr(record, 'bb_exc_info'): + etype, value, tb = record.bb_exc_info + formatted = bb.exceptions.format_exception(etype, value, tb, limit=5) + msg += '\n' + ''.join(formatted) + return msg + + def colorize(self, record): + color = self.COLORS[record.levelno] + if self.color_enabled and color is not None: + record = copy.copy(record) + record.levelname = "".join([self.BLD % color, record.levelname, self.RST]) + record.msg = "".join([self.STD % color, record.msg, self.RST]) + return record + + def enable_color(self): + self.color_enabled = True + +class BBLogFilter(object): + def __init__(self, handler, level, debug_domains): + self.stdlevel = level + self.debug_domains = debug_domains + loglevel = level + for domain in debug_domains: + if debug_domains[domain] < loglevel: + loglevel = debug_domains[domain] + handler.setLevel(loglevel) + handler.addFilter(self) + + def filter(self, record): + if record.levelno >= self.stdlevel: + return True + if record.name in self.debug_domains and record.levelno >= self.debug_domains[record.name]: + return True + return False + +class BBLogFilterStdErr(BBLogFilter): + def filter(self, record): + if not BBLogFilter.filter(self, record): + return False + if record.levelno >= logging.ERROR: + return True + return False + +class BBLogFilterStdOut(BBLogFilter): + def filter(self, record): + if not BBLogFilter.filter(self, record): + return False + if record.levelno < logging.ERROR: + return True + return False + +# Message control functions +# + +loggerDefaultDebugLevel = 0 +loggerDefaultVerbose = False +loggerVerboseLogs = False +loggerDefaultDomains = [] + +def init_msgconfig(verbose, debug, debug_domains=None): + """ + Set default verbosity and debug levels config the logger + """ + bb.msg.loggerDefaultDebugLevel = debug + bb.msg.loggerDefaultVerbose = verbose + if verbose: + bb.msg.loggerVerboseLogs = True + if debug_domains: + bb.msg.loggerDefaultDomains = debug_domains + else: + bb.msg.loggerDefaultDomains = [] + +def constructLogOptions(): + debug = loggerDefaultDebugLevel + verbose = loggerDefaultVerbose + domains = loggerDefaultDomains + + if debug: + level = BBLogFormatter.DEBUG - debug + 1 + elif verbose: + level = BBLogFormatter.VERBOSE + else: + level = BBLogFormatter.NOTE + + debug_domains = {} + for (domainarg, iterator) in groupby(domains): + dlevel = len(tuple(iterator)) + debug_domains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1 + return level, debug_domains + +def addDefaultlogFilter(handler, cls = BBLogFilter): + level, debug_domains = constructLogOptions() + + cls(handler, level, debug_domains) + +# +# Message handling functions +# + +def fatal(msgdomain, msg): + if msgdomain: + logger = logging.getLogger("BitBake.%s" % msgdomain) + else: + logger = logging.getLogger("BitBake") + logger.critical(msg) + sys.exit(1) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/namedtuple_with_abc.py b/import-layers/yocto-poky/bitbake/lib/bb/namedtuple_with_abc.py new file mode 100644 index 000000000..32f2fc642 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/namedtuple_with_abc.py @@ -0,0 +1,255 @@ +# http://code.activestate.com/recipes/577629-namedtupleabc-abstract-base-class-mix-in-for-named/ +#!/usr/bin/env python +# Copyright (c) 2011 Jan Kaliszewski (zuo). Available under the MIT License. + +""" +namedtuple_with_abc.py: +* named tuple mix-in + ABC (abstract base class) recipe, +* works under Python 2.6, 2.7 as well as 3.x. + +Import this module to patch collections.namedtuple() factory function +-- enriching it with the 'abc' attribute (an abstract base class + mix-in +for named tuples) and decorating it with a wrapper that registers each +newly created named tuple as a subclass of namedtuple.abc. + +How to import: + import collections, namedtuple_with_abc +or: + import namedtuple_with_abc + from collections import namedtuple + # ^ in this variant you must import namedtuple function + # *after* importing namedtuple_with_abc module +or simply: + from namedtuple_with_abc import namedtuple + +Simple usage example: + class Credentials(namedtuple.abc): + _fields = 'username password' + def __str__(self): + return ('{0.__class__.__name__}' + '(username={0.username}, password=...)'.format(self)) + print(Credentials("alice", "Alice's password")) + +For more advanced examples -- see below the "if __name__ == '__main__':". +""" + +import collections +from abc import ABCMeta, abstractproperty +from functools import wraps +from sys import version_info + +__all__ = ('namedtuple',) +_namedtuple = collections.namedtuple + + +class _NamedTupleABCMeta(ABCMeta): + '''The metaclass for the abstract base class + mix-in for named tuples.''' + def __new__(mcls, name, bases, namespace): + fields = namespace.get('_fields') + for base in bases: + if fields is not None: + break + fields = getattr(base, '_fields', None) + if not isinstance(fields, abstractproperty): + basetuple = _namedtuple(name, fields) + bases = (basetuple,) + bases + namespace.pop('_fields', None) + namespace.setdefault('__doc__', basetuple.__doc__) + namespace.setdefault('__slots__', ()) + return ABCMeta.__new__(mcls, name, bases, namespace) + + +exec( + # Python 2.x metaclass declaration syntax + """class _NamedTupleABC(object): + '''The abstract base class + mix-in for named tuples.''' + __metaclass__ = _NamedTupleABCMeta + _fields = abstractproperty()""" if version_info[0] < 3 else + # Python 3.x metaclass declaration syntax + """class _NamedTupleABC(metaclass=_NamedTupleABCMeta): + '''The abstract base class + mix-in for named tuples.''' + _fields = abstractproperty()""" +) + + +_namedtuple.abc = _NamedTupleABC +#_NamedTupleABC.register(type(version_info)) # (and similar, in the future...) + +@wraps(_namedtuple) +def namedtuple(*args, **kwargs): + '''Named tuple factory with namedtuple.abc subclass registration.''' + cls = _namedtuple(*args, **kwargs) + _NamedTupleABC.register(cls) + return cls + +collections.namedtuple = namedtuple + + + + +if __name__ == '__main__': + + '''Examples and explanations''' + + # Simple usage + + class MyRecord(namedtuple.abc): + _fields = 'x y z' # such form will be transformed into ('x', 'y', 'z') + def _my_custom_method(self): + return list(self._asdict().items()) + # (the '_fields' attribute belongs to the named tuple public API anyway) + + rec = MyRecord(1, 2, 3) + print(rec) + print(rec._my_custom_method()) + print(rec._replace(y=222)) + print(rec._replace(y=222)._my_custom_method()) + + # Custom abstract classes... + + class MyAbstractRecord(namedtuple.abc): + def _my_custom_method(self): + return list(self._asdict().items()) + + try: + MyAbstractRecord() # (abstract classes cannot be instantiated) + except TypeError as exc: + print(exc) + + class AnotherAbstractRecord(MyAbstractRecord): + def __str__(self): + return '<<<{0}>>>'.format(super(AnotherAbstractRecord, + self).__str__()) + + # ...and their non-abstract subclasses + + class MyRecord2(MyAbstractRecord): + _fields = 'a, b' + + class MyRecord3(AnotherAbstractRecord): + _fields = 'p', 'q', 'r' + + rec2 = MyRecord2('foo', 'bar') + print(rec2) + print(rec2._my_custom_method()) + print(rec2._replace(b=222)) + print(rec2._replace(b=222)._my_custom_method()) + + rec3 = MyRecord3('foo', 'bar', 'baz') + print(rec3) + print(rec3._my_custom_method()) + print(rec3._replace(q=222)) + print(rec3._replace(q=222)._my_custom_method()) + + # You can also subclass non-abstract ones... + + class MyRecord33(MyRecord3): + def __str__(self): + return '< {0!r}, ..., {0!r} >'.format(self.p, self.r) + + rec33 = MyRecord33('foo', 'bar', 'baz') + print(rec33) + print(rec33._my_custom_method()) + print(rec33._replace(q=222)) + print(rec33._replace(q=222)._my_custom_method()) + + # ...and even override the magic '_fields' attribute again + + class MyRecord345(MyRecord3): + _fields = 'e f g h i j k' + + rec345 = MyRecord345(1, 2, 3, 4, 3, 2, 1) + print(rec345) + print(rec345._my_custom_method()) + print(rec345._replace(f=222)) + print(rec345._replace(f=222)._my_custom_method()) + + # Mixing-in some other classes is also possible: + + class MyMixIn(object): + def method(self): + return "MyMixIn.method() called" + def _my_custom_method(self): + return "MyMixIn._my_custom_method() called" + def count(self, item): + return "MyMixIn.count({0}) called".format(item) + def _asdict(self): # (cannot override a namedtuple method, see below) + return "MyMixIn._asdict() called" + + class MyRecord4(MyRecord33, MyMixIn): # mix-in on the right + _fields = 'j k l x' + + class MyRecord5(MyMixIn, MyRecord33): # mix-in on the left + _fields = 'j k l x y' + + rec4 = MyRecord4(1, 2, 3, 2) + print(rec4) + print(rec4.method()) + print(rec4._my_custom_method()) # MyRecord33's + print(rec4.count(2)) # tuple's + print(rec4._replace(k=222)) + print(rec4._replace(k=222).method()) + print(rec4._replace(k=222)._my_custom_method()) # MyRecord33's + print(rec4._replace(k=222).count(8)) # tuple's + + rec5 = MyRecord5(1, 2, 3, 2, 1) + print(rec5) + print(rec5.method()) + print(rec5._my_custom_method()) # MyMixIn's + print(rec5.count(2)) # MyMixIn's + print(rec5._replace(k=222)) + print(rec5._replace(k=222).method()) + print(rec5._replace(k=222)._my_custom_method()) # MyMixIn's + print(rec5._replace(k=222).count(2)) # MyMixIn's + + # Note that behavior: the standard namedtuple methods cannot be + # overridden by a foreign mix-in -- even if the mix-in is declared + # as the leftmost base class (but, obviously, you can override them + # in the defined class or its subclasses): + + print(rec4._asdict()) # (returns a dict, not "MyMixIn._asdict() called") + print(rec5._asdict()) # (returns a dict, not "MyMixIn._asdict() called") + + class MyRecord6(MyRecord33): + _fields = 'j k l x y z' + def _asdict(self): + return "MyRecord6._asdict() called" + rec6 = MyRecord6(1, 2, 3, 1, 2, 3) + print(rec6._asdict()) # (this returns "MyRecord6._asdict() called") + + # All that record classes are real subclasses of namedtuple.abc: + + assert issubclass(MyRecord, namedtuple.abc) + assert issubclass(MyAbstractRecord, namedtuple.abc) + assert issubclass(AnotherAbstractRecord, namedtuple.abc) + assert issubclass(MyRecord2, namedtuple.abc) + assert issubclass(MyRecord3, namedtuple.abc) + assert issubclass(MyRecord33, namedtuple.abc) + assert issubclass(MyRecord345, namedtuple.abc) + assert issubclass(MyRecord4, namedtuple.abc) + assert issubclass(MyRecord5, namedtuple.abc) + assert issubclass(MyRecord6, namedtuple.abc) + + # ...but abstract ones are not subclasses of tuple + # (and this is what you probably want): + + assert not issubclass(MyAbstractRecord, tuple) + assert not issubclass(AnotherAbstractRecord, tuple) + + assert issubclass(MyRecord, tuple) + assert issubclass(MyRecord2, tuple) + assert issubclass(MyRecord3, tuple) + assert issubclass(MyRecord33, tuple) + assert issubclass(MyRecord345, tuple) + assert issubclass(MyRecord4, tuple) + assert issubclass(MyRecord5, tuple) + assert issubclass(MyRecord6, tuple) + + # Named tuple classes created with namedtuple() factory function + # (in the "traditional" way) are registered as "virtual" subclasses + # of namedtuple.abc: + + MyTuple = namedtuple('MyTuple', 'a b c') + mt = MyTuple(1, 2, 3) + assert issubclass(MyTuple, namedtuple.abc) + assert isinstance(mt, namedtuple.abc) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py new file mode 100644 index 000000000..26ae7ead8 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py @@ -0,0 +1,170 @@ +""" +BitBake Parsers + +File parsers for the BitBake build tools. + +""" + + +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +handlers = [] + +import errno +import logging +import os +import stat +import bb +import bb.utils +import bb.siggen + +logger = logging.getLogger("BitBake.Parsing") + +class ParseError(Exception): + """Exception raised when parsing fails""" + def __init__(self, msg, filename, lineno=0): + self.msg = msg + self.filename = filename + self.lineno = lineno + Exception.__init__(self, msg, filename, lineno) + + def __str__(self): + if self.lineno: + return "ParseError at %s:%d: %s" % (self.filename, self.lineno, self.msg) + else: + return "ParseError in %s: %s" % (self.filename, self.msg) + +class SkipRecipe(Exception): + """Exception raised to skip this recipe""" + +class SkipPackage(SkipRecipe): + """Exception raised to skip this recipe (use SkipRecipe in new code)""" + +__mtime_cache = {} +def cached_mtime(f): + if f not in __mtime_cache: + __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] + return __mtime_cache[f] + +def cached_mtime_noerror(f): + if f not in __mtime_cache: + try: + __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] + except OSError: + return 0 + return __mtime_cache[f] + +def update_mtime(f): + try: + __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] + except OSError: + if f in __mtime_cache: + del __mtime_cache[f] + return 0 + return __mtime_cache[f] + +def update_cache(f): + if f in __mtime_cache: + logger.debug(1, "Updating mtime cache for %s" % f) + update_mtime(f) + +def mark_dependency(d, f): + if f.startswith('./'): + f = "%s/%s" % (os.getcwd(), f[2:]) + deps = (d.getVar('__depends', False) or []) + s = (f, cached_mtime_noerror(f)) + if s not in deps: + deps.append(s) + d.setVar('__depends', deps) + +def check_dependency(d, f): + s = (f, cached_mtime_noerror(f)) + deps = (d.getVar('__depends', False) or []) + return s in deps + +def supports(fn, data): + """Returns true if we have a handler for this file, false otherwise""" + for h in handlers: + if h['supports'](fn, data): + return 1 + return 0 + +def handle(fn, data, include = 0): + """Call the handler that is appropriate for this file""" + for h in handlers: + if h['supports'](fn, data): + with data.inchistory.include(fn): + return h['handle'](fn, data, include) + raise ParseError("not a BitBake file", fn) + +def init(fn, data): + for h in handlers: + if h['supports'](fn): + return h['init'](data) + +def init_parser(d): + bb.parse.siggen = bb.siggen.init(d) + +def resolve_file(fn, d): + if not os.path.isabs(fn): + bbpath = d.getVar("BBPATH", True) + newfn, attempts = bb.utils.which(bbpath, fn, history=True) + for af in attempts: + mark_dependency(d, af) + if not newfn: + raise IOError(errno.ENOENT, "file %s not found in %s" % (fn, bbpath)) + fn = newfn + + mark_dependency(d, fn) + if not os.path.isfile(fn): + raise IOError(errno.ENOENT, "file %s not found" % fn) + + return fn + +# Used by OpenEmbedded metadata +__pkgsplit_cache__={} +def vars_from_file(mypkg, d): + if not mypkg or not mypkg.endswith((".bb", ".bbappend")): + return (None, None, None) + if mypkg in __pkgsplit_cache__: + return __pkgsplit_cache__[mypkg] + + myfile = os.path.splitext(os.path.basename(mypkg)) + parts = myfile[0].split('_') + __pkgsplit_cache__[mypkg] = parts + if len(parts) > 3: + raise ParseError("Unable to generate default variables from filename (too many underscores)", mypkg) + exp = 3 - len(parts) + tmplist = [] + while exp != 0: + exp -= 1 + tmplist.append(None) + parts.extend(tmplist) + return parts + +def get_file_depends(d): + '''Return the dependent files''' + dep_files = [] + depends = d.getVar('__base_depends', False) or [] + depends = depends + (d.getVar('__depends', False) or []) + for (fn, _) in depends: + dep_files.append(os.path.abspath(fn)) + return " ".join(dep_files) + +from bb.parse.parse_py import __version__, ConfHandler, BBHandler diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py new file mode 100644 index 000000000..5f55af5ef --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py @@ -0,0 +1,476 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" + AbstractSyntaxTree classes for the Bitbake language +""" + +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2009 Holger Hans Peter Freyther +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import absolute_import +from future_builtins import filter +import re +import string +import logging +import bb +import itertools +from bb import methodpool +from bb.parse import logger + +_bbversions_re = re.compile(r"\[(?P<from>[0-9]+)-(?P<to>[0-9]+)\]") + +class StatementGroup(list): + def eval(self, data): + for statement in self: + statement.eval(data) + +class AstNode(object): + def __init__(self, filename, lineno): + self.filename = filename + self.lineno = lineno + +class IncludeNode(AstNode): + def __init__(self, filename, lineno, what_file, force): + AstNode.__init__(self, filename, lineno) + self.what_file = what_file + self.force = force + + def eval(self, data): + """ + Include the file and evaluate the statements + """ + s = data.expand(self.what_file) + logger.debug(2, "CONF %s:%s: including %s", self.filename, self.lineno, s) + + # TODO: Cache those includes... maybe not here though + if self.force: + bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, "include required") + else: + bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, False) + +class ExportNode(AstNode): + def __init__(self, filename, lineno, var): + AstNode.__init__(self, filename, lineno) + self.var = var + + def eval(self, data): + data.setVarFlag(self.var, "export", 1, op = 'exported') + +class DataNode(AstNode): + """ + Various data related updates. For the sake of sanity + we have one class doing all this. This means that all + this need to be re-evaluated... we might be able to do + that faster with multiple classes. + """ + def __init__(self, filename, lineno, groupd): + AstNode.__init__(self, filename, lineno) + self.groupd = groupd + + def getFunc(self, key, data): + if 'flag' in self.groupd and self.groupd['flag'] != None: + return data.getVarFlag(key, self.groupd['flag'], expand=False, noweakdefault=True) + else: + return data.getVar(key, False, noweakdefault=True, parsing=True) + + def eval(self, data): + groupd = self.groupd + key = groupd["var"] + loginfo = { + 'variable': key, + 'file': self.filename, + 'line': self.lineno, + } + if "exp" in groupd and groupd["exp"] != None: + data.setVarFlag(key, "export", 1, op = 'exported', **loginfo) + + op = "set" + if "ques" in groupd and groupd["ques"] != None: + val = self.getFunc(key, data) + op = "set?" + if val == None: + val = groupd["value"] + elif "colon" in groupd and groupd["colon"] != None: + e = data.createCopy() + bb.data.update_data(e) + op = "immediate" + val = e.expand(groupd["value"], key + "[:=]") + elif "append" in groupd and groupd["append"] != None: + op = "append" + val = "%s %s" % ((self.getFunc(key, data) or ""), groupd["value"]) + elif "prepend" in groupd and groupd["prepend"] != None: + op = "prepend" + val = "%s %s" % (groupd["value"], (self.getFunc(key, data) or "")) + elif "postdot" in groupd and groupd["postdot"] != None: + op = "postdot" + val = "%s%s" % ((self.getFunc(key, data) or ""), groupd["value"]) + elif "predot" in groupd and groupd["predot"] != None: + op = "predot" + val = "%s%s" % (groupd["value"], (self.getFunc(key, data) or "")) + else: + val = groupd["value"] + + flag = None + if 'flag' in groupd and groupd['flag'] != None: + flag = groupd['flag'] + elif groupd["lazyques"]: + flag = "_defaultval" + + loginfo['op'] = op + loginfo['detail'] = groupd["value"] + + if flag: + data.setVarFlag(key, flag, val, **loginfo) + else: + data.setVar(key, val, parsing=True, **loginfo) + +class MethodNode(AstNode): + tr_tbl = string.maketrans('/.+-@%&', '_______') + + def __init__(self, filename, lineno, func_name, body, python, fakeroot): + AstNode.__init__(self, filename, lineno) + self.func_name = func_name + self.body = body + self.python = python + self.fakeroot = fakeroot + + def eval(self, data): + text = '\n'.join(self.body) + funcname = self.func_name + if self.func_name == "__anonymous": + funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl))) + self.python = True + text = "def %s(d):\n" % (funcname) + text + bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body)) + anonfuncs = data.getVar('__BBANONFUNCS', False) or [] + anonfuncs.append(funcname) + data.setVar('__BBANONFUNCS', anonfuncs) + if data.getVar(funcname, False): + # clean up old version of this piece of metadata, as its + # flags could cause problems + data.delVarFlag(funcname, 'python') + data.delVarFlag(funcname, 'fakeroot') + if self.python: + data.setVarFlag(funcname, "python", "1") + if self.fakeroot: + data.setVarFlag(funcname, "fakeroot", "1") + data.setVarFlag(funcname, "func", 1) + data.setVar(funcname, text, parsing=True) + data.setVarFlag(funcname, 'filename', self.filename) + data.setVarFlag(funcname, 'lineno', str(self.lineno - len(self.body))) + +class PythonMethodNode(AstNode): + def __init__(self, filename, lineno, function, modulename, body): + AstNode.__init__(self, filename, lineno) + self.function = function + self.modulename = modulename + self.body = body + + def eval(self, data): + # Note we will add root to parsedmethods after having parse + # 'this' file. This means we will not parse methods from + # bb classes twice + text = '\n'.join(self.body) + bb.methodpool.insert_method(self.modulename, text, self.filename, self.lineno - len(self.body) - 1) + data.setVarFlag(self.function, "func", 1) + data.setVarFlag(self.function, "python", 1) + data.setVar(self.function, text, parsing=True) + data.setVarFlag(self.function, 'filename', self.filename) + data.setVarFlag(self.function, 'lineno', str(self.lineno - len(self.body) - 1)) + +class ExportFuncsNode(AstNode): + def __init__(self, filename, lineno, fns, classname): + AstNode.__init__(self, filename, lineno) + self.n = fns.split() + self.classname = classname + + def eval(self, data): + + for func in self.n: + calledfunc = self.classname + "_" + func + + if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False): + continue + + if data.getVar(func, False): + data.setVarFlag(func, 'python', None) + data.setVarFlag(func, 'func', None) + + for flag in [ "func", "python" ]: + if data.getVarFlag(calledfunc, flag, False): + data.setVarFlag(func, flag, data.getVarFlag(calledfunc, flag, False)) + for flag in [ "dirs" ]: + if data.getVarFlag(func, flag, False): + data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag, False)) + data.setVarFlag(func, "filename", "autogenerated") + data.setVarFlag(func, "lineno", 1) + + if data.getVarFlag(calledfunc, "python", False): + data.setVar(func, " bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True) + else: + if "-" in self.classname: + bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc)) + data.setVar(func, " " + calledfunc + "\n", parsing=True) + data.setVarFlag(func, 'export_func', '1') + +class AddTaskNode(AstNode): + def __init__(self, filename, lineno, func, before, after): + AstNode.__init__(self, filename, lineno) + self.func = func + self.before = before + self.after = after + + def eval(self, data): + bb.build.addtask(self.func, self.before, self.after, data) + +class DelTaskNode(AstNode): + def __init__(self, filename, lineno, func): + AstNode.__init__(self, filename, lineno) + self.func = func + + def eval(self, data): + bb.build.deltask(self.func, data) + +class BBHandlerNode(AstNode): + def __init__(self, filename, lineno, fns): + AstNode.__init__(self, filename, lineno) + self.hs = fns.split() + + def eval(self, data): + bbhands = data.getVar('__BBHANDLERS', False) or [] + for h in self.hs: + bbhands.append(h) + data.setVarFlag(h, "handler", 1) + data.setVar('__BBHANDLERS', bbhands) + +class InheritNode(AstNode): + def __init__(self, filename, lineno, classes): + AstNode.__init__(self, filename, lineno) + self.classes = classes + + def eval(self, data): + bb.parse.BBHandler.inherit(self.classes, self.filename, self.lineno, data) + +def handleInclude(statements, filename, lineno, m, force): + statements.append(IncludeNode(filename, lineno, m.group(1), force)) + +def handleExport(statements, filename, lineno, m): + statements.append(ExportNode(filename, lineno, m.group(1))) + +def handleData(statements, filename, lineno, groupd): + statements.append(DataNode(filename, lineno, groupd)) + +def handleMethod(statements, filename, lineno, func_name, body, python, fakeroot): + statements.append(MethodNode(filename, lineno, func_name, body, python, fakeroot)) + +def handlePythonMethod(statements, filename, lineno, funcname, modulename, body): + statements.append(PythonMethodNode(filename, lineno, funcname, modulename, body)) + +def handleExportFuncs(statements, filename, lineno, m, classname): + statements.append(ExportFuncsNode(filename, lineno, m.group(1), classname)) + +def handleAddTask(statements, filename, lineno, m): + func = m.group("func") + before = m.group("before") + after = m.group("after") + if func is None: + return + + statements.append(AddTaskNode(filename, lineno, func, before, after)) + +def handleDelTask(statements, filename, lineno, m): + func = m.group("func") + if func is None: + return + + statements.append(DelTaskNode(filename, lineno, func)) + +def handleBBHandlers(statements, filename, lineno, m): + statements.append(BBHandlerNode(filename, lineno, m.group(1))) + +def handleInherit(statements, filename, lineno, m): + classes = m.group(1) + statements.append(InheritNode(filename, lineno, classes)) + +def finalize(fn, d, variant = None): + all_handlers = {} + for var in d.getVar('__BBHANDLERS', False) or []: + # try to add the handler + handlerfn = d.getVarFlag(var, "filename", False) + handlerln = int(d.getVarFlag(var, "lineno", False)) + bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln) + + bb.event.fire(bb.event.RecipePreFinalise(fn), d) + + bb.data.expandKeys(d) + bb.data.update_data(d) + code = [] + for funcname in d.getVar("__BBANONFUNCS", False) or []: + code.append("%s(d)" % funcname) + bb.utils.better_exec("\n".join(code), {"d": d}) + bb.data.update_data(d) + + tasklist = d.getVar('__BBTASKS', False) or [] + bb.build.add_tasks(tasklist, d) + + bb.parse.siggen.finalise(fn, d, variant) + + d.setVar('BBINCLUDED', bb.parse.get_file_depends(d)) + + bb.event.fire(bb.event.RecipeParsed(fn), d) + +def _create_variants(datastores, names, function, onlyfinalise): + def create_variant(name, orig_d, arg = None): + if onlyfinalise and name not in onlyfinalise: + return + new_d = bb.data.createCopy(orig_d) + function(arg or name, new_d) + datastores[name] = new_d + + for variant, variant_d in datastores.items(): + for name in names: + if not variant: + # Based on main recipe + create_variant(name, variant_d) + else: + create_variant("%s-%s" % (variant, name), variant_d, name) + +def _expand_versions(versions): + def expand_one(version, start, end): + for i in xrange(start, end + 1): + ver = _bbversions_re.sub(str(i), version, 1) + yield ver + + versions = iter(versions) + while True: + try: + version = next(versions) + except StopIteration: + break + + range_ver = _bbversions_re.search(version) + if not range_ver: + yield version + else: + newversions = expand_one(version, int(range_ver.group("from")), + int(range_ver.group("to"))) + versions = itertools.chain(newversions, versions) + +def multi_finalize(fn, d): + appends = (d.getVar("__BBAPPEND", True) or "").split() + for append in appends: + logger.debug(1, "Appending .bbappend file %s to %s", append, fn) + bb.parse.BBHandler.handle(append, d, True) + + onlyfinalise = d.getVar("__ONLYFINALISE", False) + + safe_d = d + d = bb.data.createCopy(safe_d) + try: + finalize(fn, d) + except bb.parse.SkipRecipe as e: + d.setVar("__SKIPPED", e.args[0]) + datastores = {"": safe_d} + + versions = (d.getVar("BBVERSIONS", True) or "").split() + if versions: + pv = orig_pv = d.getVar("PV", True) + baseversions = {} + + def verfunc(ver, d, pv_d = None): + if pv_d is None: + pv_d = d + + overrides = d.getVar("OVERRIDES", True).split(":") + pv_d.setVar("PV", ver) + overrides.append(ver) + bpv = baseversions.get(ver) or orig_pv + pv_d.setVar("BPV", bpv) + overrides.append(bpv) + d.setVar("OVERRIDES", ":".join(overrides)) + + versions = list(_expand_versions(versions)) + for pos, version in enumerate(list(versions)): + try: + pv, bpv = version.split(":", 2) + except ValueError: + pass + else: + versions[pos] = pv + baseversions[pv] = bpv + + if pv in versions and not baseversions.get(pv): + versions.remove(pv) + else: + pv = versions.pop() + + # This is necessary because our existing main datastore + # has already been finalized with the old PV, we need one + # that's been finalized with the new PV. + d = bb.data.createCopy(safe_d) + verfunc(pv, d, safe_d) + try: + finalize(fn, d) + except bb.parse.SkipRecipe as e: + d.setVar("__SKIPPED", e.args[0]) + + _create_variants(datastores, versions, verfunc, onlyfinalise) + + extended = d.getVar("BBCLASSEXTEND", True) or "" + if extended: + # the following is to support bbextends with arguments, for e.g. multilib + # an example is as follows: + # BBCLASSEXTEND = "multilib:lib32" + # it will create foo-lib32, inheriting multilib.bbclass and set + # BBEXTENDCURR to "multilib" and BBEXTENDVARIANT to "lib32" + extendedmap = {} + variantmap = {} + + for ext in extended.split(): + eext = ext.split(':', 2) + if len(eext) > 1: + extendedmap[ext] = eext[0] + variantmap[ext] = eext[1] + else: + extendedmap[ext] = ext + + pn = d.getVar("PN", True) + def extendfunc(name, d): + if name != extendedmap[name]: + d.setVar("BBEXTENDCURR", extendedmap[name]) + d.setVar("BBEXTENDVARIANT", variantmap[name]) + else: + d.setVar("PN", "%s-%s" % (pn, name)) + bb.parse.BBHandler.inherit(extendedmap[name], fn, 0, d) + + safe_d.setVar("BBCLASSEXTEND", extended) + _create_variants(datastores, extendedmap.keys(), extendfunc, onlyfinalise) + + for variant, variant_d in datastores.iteritems(): + if variant: + try: + if not onlyfinalise or variant in onlyfinalise: + finalize(fn, variant_d, variant) + except bb.parse.SkipRecipe as e: + variant_d.setVar("__SKIPPED", e.args[0]) + + if len(datastores) > 1: + variants = filter(None, datastores.iterkeys()) + safe_d.setVar("__VARIANTS", " ".join(variants)) + + datastores[""] = d + return datastores diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py new file mode 100644 index 000000000..ef72c3700 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" + class for handling .bb files + + Reads a .bb file and obtains its metadata + +""" + + +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import absolute_import +import re, bb, os +import logging +import bb.build, bb.utils +from bb import data + +from . import ConfHandler +from .. import resolve_file, ast, logger, ParseError +from .ConfHandler import include, init + +# For compatibility +bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"]) + +__func_start_regexp__ = re.compile( r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*(?P<func>[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" ) +__inherit_regexp__ = re.compile( r"inherit\s+(.+)" ) +__export_func_regexp__ = re.compile( r"EXPORT_FUNCTIONS\s+(.+)" ) +__addtask_regexp__ = re.compile("addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*") +__deltask_regexp__ = re.compile("deltask\s+(?P<func>\w+)") +__addhandler_regexp__ = re.compile( r"addhandler\s+(.+)" ) +__def_regexp__ = re.compile( r"def\s+(\w+).*:" ) +__python_func_regexp__ = re.compile( r"(\s+.*)|(^$)" ) + +__infunc__ = [] +__inpython__ = False +__body__ = [] +__classname__ = "" + +cached_statements = {} + +def supports(fn, d): + """Return True if fn has a supported extension""" + return os.path.splitext(fn)[-1] in [".bb", ".bbclass", ".inc"] + +def inherit(files, fn, lineno, d): + __inherit_cache = d.getVar('__inherit_cache', False) or [] + files = d.expand(files).split() + for file in files: + if not os.path.isabs(file) and not file.endswith(".bbclass"): + file = os.path.join('classes', '%s.bbclass' % file) + + if not os.path.isabs(file): + bbpath = d.getVar("BBPATH", True) + abs_fn, attempts = bb.utils.which(bbpath, file, history=True) + for af in attempts: + if af != abs_fn: + bb.parse.mark_dependency(d, af) + if abs_fn: + file = abs_fn + + if not file in __inherit_cache: + logger.debug(1, "Inheriting %s (from %s:%d)" % (file, fn, lineno)) + __inherit_cache.append( file ) + d.setVar('__inherit_cache', __inherit_cache) + include(fn, file, lineno, d, "inherit") + __inherit_cache = d.getVar('__inherit_cache', False) or [] + +def get_statements(filename, absolute_filename, base_name): + global cached_statements + + try: + return cached_statements[absolute_filename] + except KeyError: + file = open(absolute_filename, 'r') + statements = ast.StatementGroup() + + lineno = 0 + while True: + lineno = lineno + 1 + s = file.readline() + if not s: break + s = s.rstrip() + feeder(lineno, s, filename, base_name, statements) + file.close() + if __inpython__: + # add a blank line to close out any python definition + feeder(lineno, "", filename, base_name, statements, eof=True) + + if filename.endswith(".bbclass") or filename.endswith(".inc"): + cached_statements[absolute_filename] = statements + return statements + +def handle(fn, d, include): + global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __residue__, __classname__ + __body__ = [] + __infunc__ = [] + __classname__ = "" + __residue__ = [] + + base_name = os.path.basename(fn) + (root, ext) = os.path.splitext(base_name) + init(d) + + if ext == ".bbclass": + __classname__ = root + __inherit_cache = d.getVar('__inherit_cache', False) or [] + if not fn in __inherit_cache: + __inherit_cache.append(fn) + d.setVar('__inherit_cache', __inherit_cache) + + if include != 0: + oldfile = d.getVar('FILE', False) + else: + oldfile = None + + abs_fn = resolve_file(fn, d) + + if include: + bb.parse.mark_dependency(d, abs_fn) + + # actual loading + statements = get_statements(fn, abs_fn, base_name) + + # DONE WITH PARSING... time to evaluate + if ext != ".bbclass" and abs_fn != oldfile: + d.setVar('FILE', abs_fn) + + try: + statements.eval(d) + except bb.parse.SkipRecipe: + bb.data.setVar("__SKIPPED", True, d) + if include == 0: + return { "" : d } + + if __infunc__: + raise ParseError("Shell function %s is never closed" % __infunc__[0], __infunc__[1], __infunc__[2]) + if __residue__: + raise ParseError("Leftover unparsed (incomplete?) data %s from %s" % __residue__, fn) + + if ext != ".bbclass" and include == 0: + return ast.multi_finalize(fn, d) + + if ext != ".bbclass" and oldfile and abs_fn != oldfile: + d.setVar("FILE", oldfile) + + return d + +def feeder(lineno, s, fn, root, statements, eof=False): + global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__, __infunc__, __body__, bb, __residue__, __classname__ + if __infunc__: + if s == '}': + __body__.append('') + ast.handleMethod(statements, fn, lineno, __infunc__[0], __body__, __infunc__[3], __infunc__[4]) + __infunc__ = [] + __body__ = [] + else: + __body__.append(s) + return + + if __inpython__: + m = __python_func_regexp__.match(s) + if m and not eof: + __body__.append(s) + return + else: + ast.handlePythonMethod(statements, fn, lineno, __inpython__, + root, __body__) + __body__ = [] + __inpython__ = False + + if eof: + return + + if s and s[0] == '#': + if len(__residue__) != 0 and __residue__[0][0] != "#": + bb.fatal("There is a comment on line %s of file %s (%s) which is in the middle of a multiline expression.\nBitbake used to ignore these but no longer does so, please fix your metadata as errors are likely as a result of this change." % (lineno, fn, s)) + + if len(__residue__) != 0 and __residue__[0][0] == "#" and (not s or s[0] != "#"): + bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s)) + + if s and s[-1] == '\\': + __residue__.append(s[:-1]) + return + + s = "".join(__residue__) + s + __residue__ = [] + + # Skip empty lines + if s == '': + return + + # Skip comments + if s[0] == '#': + return + + m = __func_start_regexp__.match(s) + if m: + __infunc__ = [m.group("func") or "__anonymous", fn, lineno, m.group("py") is not None, m.group("fr") is not None] + return + + m = __def_regexp__.match(s) + if m: + __body__.append(s) + __inpython__ = m.group(1) + + return + + m = __export_func_regexp__.match(s) + if m: + ast.handleExportFuncs(statements, fn, lineno, m, __classname__) + return + + m = __addtask_regexp__.match(s) + if m: + ast.handleAddTask(statements, fn, lineno, m) + return + + m = __deltask_regexp__.match(s) + if m: + ast.handleDelTask(statements, fn, lineno, m) + return + + m = __addhandler_regexp__.match(s) + if m: + ast.handleBBHandlers(statements, fn, lineno, m) + return + + m = __inherit_regexp__.match(s) + if m: + ast.handleInherit(statements, fn, lineno, m) + return + + return ConfHandler.feeder(lineno, s, fn, statements) + +# Add us to the handlers list +from .. import handlers +handlers.append({'supports': supports, 'handle': handle, 'init': init}) +del handlers diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py new file mode 100644 index 000000000..fbd75b14a --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" + class for handling configuration data files + + Reads a .conf file and obtains its metadata + +""" + +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import errno +import re +import os +import bb.utils +from bb.parse import ParseError, resolve_file, ast, logger, handle + +__config_regexp__ = re.compile( r""" + ^ + (?P<exp>export\s*)? + (?P<var>[a-zA-Z0-9\-~_+.${}/]+?) + (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])? + + \s* ( + (?P<colon>:=) | + (?P<lazyques>\?\?=) | + (?P<ques>\?=) | + (?P<append>\+=) | + (?P<prepend>=\+) | + (?P<predot>=\.) | + (?P<postdot>\.=) | + = + ) \s* + + (?!'[^']*'[^']*'$) + (?!\"[^\"]*\"[^\"]*\"$) + (?P<apo>['\"]) + (?P<value>.*) + (?P=apo) + $ + """, re.X) +__include_regexp__ = re.compile( r"include\s+(.+)" ) +__require_regexp__ = re.compile( r"require\s+(.+)" ) +__export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/]+)$" ) + +def init(data): + topdir = data.getVar('TOPDIR', False) + if not topdir: + data.setVar('TOPDIR', os.getcwd()) + + +def supports(fn, d): + return fn[-5:] == ".conf" + +def include(parentfn, fn, lineno, data, error_out): + """ + error_out: A string indicating the verb (e.g. "include", "inherit") to be + used in a ParseError that will be raised if the file to be included could + not be included. Specify False to avoid raising an error in this case. + """ + if parentfn == fn: # prevent infinite recursion + return None + + fn = data.expand(fn) + parentfn = data.expand(parentfn) + + if not os.path.isabs(fn): + dname = os.path.dirname(parentfn) + bbpath = "%s:%s" % (dname, data.getVar("BBPATH", True)) + abs_fn, attempts = bb.utils.which(bbpath, fn, history=True) + if abs_fn and bb.parse.check_dependency(data, abs_fn): + logger.warn("Duplicate inclusion for %s in %s" % (abs_fn, data.getVar('FILE', True))) + for af in attempts: + bb.parse.mark_dependency(data, af) + if abs_fn: + fn = abs_fn + elif bb.parse.check_dependency(data, fn): + logger.warn("Duplicate inclusion for %s in %s" % (fn, data.getVar('FILE', True))) + + try: + bb.parse.handle(fn, data, True) + except (IOError, OSError) as exc: + if exc.errno == errno.ENOENT: + if error_out: + raise ParseError("Could not %s file %s" % (error_out, fn), parentfn, lineno) + logger.debug(2, "CONF file '%s' not found", fn) + else: + if error_out: + raise ParseError("Could not %s file %s: %s" % (error_out, fn, exc.strerror), parentfn, lineno) + else: + raise ParseError("Error parsing %s: %s" % (fn, exc.strerror), parentfn, lineno) + +# We have an issue where a UI might want to enforce particular settings such as +# an empty DISTRO variable. If configuration files do something like assigning +# a weak default, it turns out to be very difficult to filter out these changes, +# particularly when the weak default might appear half way though parsing a chain +# of configuration files. We therefore let the UIs hook into configuration file +# parsing. This turns out to be a hard problem to solve any other way. +confFilters = [] + +def handle(fn, data, include): + init(data) + + if include == 0: + oldfile = None + else: + oldfile = data.getVar('FILE', False) + + abs_fn = resolve_file(fn, data) + f = open(abs_fn, 'r') + + if include: + bb.parse.mark_dependency(data, abs_fn) + + statements = ast.StatementGroup() + lineno = 0 + while True: + lineno = lineno + 1 + s = f.readline() + if not s: + break + w = s.strip() + # skip empty lines + if not w: + continue + s = s.rstrip() + while s[-1] == '\\': + s2 = f.readline().strip() + lineno = lineno + 1 + if (not s2 or s2 and s2[0] != "#") and s[0] == "#" : + bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s)) + s = s[:-1] + s2 + # skip comments + if s[0] == '#': + continue + feeder(lineno, s, abs_fn, statements) + + # DONE WITH PARSING... time to evaluate + data.setVar('FILE', abs_fn) + statements.eval(data) + if oldfile: + data.setVar('FILE', oldfile) + + f.close() + + for f in confFilters: + f(fn, data) + + return data + +def feeder(lineno, s, fn, statements): + m = __config_regexp__.match(s) + if m: + groupd = m.groupdict() + ast.handleData(statements, fn, lineno, groupd) + return + + m = __include_regexp__.match(s) + if m: + ast.handleInclude(statements, fn, lineno, m, False) + return + + m = __require_regexp__.match(s) + if m: + ast.handleInclude(statements, fn, lineno, m, True) + return + + m = __export_regexp__.match(s) + if m: + ast.handleExport(statements, fn, lineno, m) + return + + raise ParseError("unparsed line: '%s'" % s, fn, lineno); + +# Add us to the handlers list +from bb.parse import handlers +handlers.append({'supports': supports, 'handle': handle, 'init': init}) +del handlers diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/__init__.py new file mode 100644 index 000000000..3e658d0de --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/__init__.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake Parsers + +File parsers for the BitBake build tools. + +""" + +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +from __future__ import absolute_import +from . import ConfHandler +from . import BBHandler + +__version__ = '1.0' diff --git a/import-layers/yocto-poky/bitbake/lib/bb/persist_data.py b/import-layers/yocto-poky/bitbake/lib/bb/persist_data.py new file mode 100644 index 000000000..e45042324 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/persist_data.py @@ -0,0 +1,218 @@ +"""BitBake Persistent Data Store + +Used to store data in a central location such that other threads/tasks can +access them at some future date. Acts as a convenience wrapper around sqlite, +currently, providing a key/value store accessed by 'domain'. +""" + +# Copyright (C) 2007 Richard Purdie +# Copyright (C) 2010 Chris Larson <chris_larson@mentor.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import collections +import logging +import os.path +import sys +import warnings +from bb.compat import total_ordering +from collections import Mapping + +try: + import sqlite3 +except ImportError: + from pysqlite2 import dbapi2 as sqlite3 + +sqlversion = sqlite3.sqlite_version_info +if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): + raise Exception("sqlite3 version 3.3.0 or later is required.") + + +logger = logging.getLogger("BitBake.PersistData") +if hasattr(sqlite3, 'enable_shared_cache'): + try: + sqlite3.enable_shared_cache(True) + except sqlite3.OperationalError: + pass + + +@total_ordering +class SQLTable(collections.MutableMapping): + """Object representing a table/domain in the database""" + def __init__(self, cachefile, table): + self.cachefile = cachefile + self.table = table + self.cursor = connect(self.cachefile) + + self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" + % table) + + def _execute(self, *query): + """Execute a query, waiting to acquire a lock if necessary""" + count = 0 + while True: + try: + return self.cursor.execute(*query) + except sqlite3.OperationalError as exc: + if 'database is locked' in str(exc) and count < 500: + count = count + 1 + self.cursor.close() + self.cursor = connect(self.cachefile) + continue + raise + + def __enter__(self): + self.cursor.__enter__() + return self + + def __exit__(self, *excinfo): + self.cursor.__exit__(*excinfo) + + def __getitem__(self, key): + data = self._execute("SELECT * from %s where key=?;" % + self.table, [key]) + for row in data: + return row[1] + raise KeyError(key) + + def __delitem__(self, key): + if key not in self: + raise KeyError(key) + self._execute("DELETE from %s where key=?;" % self.table, [key]) + + def __setitem__(self, key, value): + if not isinstance(key, basestring): + raise TypeError('Only string keys are supported') + elif not isinstance(value, basestring): + raise TypeError('Only string values are supported') + + data = self._execute("SELECT * from %s where key=?;" % + self.table, [key]) + exists = len(list(data)) + if exists: + self._execute("UPDATE %s SET value=? WHERE key=?;" % self.table, + [value, key]) + else: + self._execute("INSERT into %s(key, value) values (?, ?);" % + self.table, [key, value]) + + def __contains__(self, key): + return key in set(self) + + def __len__(self): + data = self._execute("SELECT COUNT(key) FROM %s;" % self.table) + for row in data: + return row[0] + + def __iter__(self): + data = self._execute("SELECT key FROM %s;" % self.table) + return (row[0] for row in data) + + def __lt__(self, other): + if not isinstance(other, Mapping): + raise NotImplemented + + return len(self) < len(other) + + def get_by_pattern(self, pattern): + data = self._execute("SELECT * FROM %s WHERE key LIKE ?;" % + self.table, [pattern]) + return [row[1] for row in data] + + def values(self): + return list(self.itervalues()) + + def itervalues(self): + data = self._execute("SELECT value FROM %s;" % self.table) + return (row[0] for row in data) + + def items(self): + return list(self.iteritems()) + + def iteritems(self): + return self._execute("SELECT * FROM %s;" % self.table) + + def clear(self): + self._execute("DELETE FROM %s;" % self.table) + + def has_key(self, key): + return key in self + + +class PersistData(object): + """Deprecated representation of the bitbake persistent data store""" + def __init__(self, d): + warnings.warn("Use of PersistData is deprecated. Please use " + "persist(domain, d) instead.", + category=DeprecationWarning, + stacklevel=2) + + self.data = persist(d) + logger.debug(1, "Using '%s' as the persistent data cache", + self.data.filename) + + def addDomain(self, domain): + """ + Add a domain (pending deprecation) + """ + return self.data[domain] + + def delDomain(self, domain): + """ + Removes a domain and all the data it contains + """ + del self.data[domain] + + def getKeyValues(self, domain): + """ + Return a list of key + value pairs for a domain + """ + return self.data[domain].items() + + def getValue(self, domain, key): + """ + Return the value of a key for a domain + """ + return self.data[domain][key] + + def setValue(self, domain, key, value): + """ + Sets the value of a key for a domain + """ + self.data[domain][key] = value + + def delValue(self, domain, key): + """ + Deletes a key/value pair + """ + del self.data[domain][key] + +def connect(database): + connection = sqlite3.connect(database, timeout=5, isolation_level=None) + connection.execute("pragma synchronous = off;") + connection.text_factory = str + return connection + +def persist(domain, d): + """Convenience factory for SQLTable objects based upon metadata""" + import bb.utils + cachedir = (d.getVar("PERSISTENT_DIR", True) or + d.getVar("CACHE", True)) + if not cachedir: + logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable") + sys.exit(1) + + bb.utils.mkdirhier(cachedir) + cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3") + return SQLTable(cachefile, domain) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/process.py b/import-layers/yocto-poky/bitbake/lib/bb/process.py new file mode 100644 index 000000000..1c07f2d9b --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/process.py @@ -0,0 +1,156 @@ +import logging +import signal +import subprocess +import errno +import select + +logger = logging.getLogger('BitBake.Process') + +def subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +class CmdError(RuntimeError): + def __init__(self, command, msg=None): + self.command = command + self.msg = msg + + def __str__(self): + if not isinstance(self.command, basestring): + cmd = subprocess.list2cmdline(self.command) + else: + cmd = self.command + + msg = "Execution of '%s' failed" % cmd + if self.msg: + msg += ': %s' % self.msg + return msg + +class NotFoundError(CmdError): + def __str__(self): + return CmdError.__str__(self) + ": command not found" + +class ExecutionError(CmdError): + def __init__(self, command, exitcode, stdout = None, stderr = None): + CmdError.__init__(self, command) + self.exitcode = exitcode + self.stdout = stdout + self.stderr = stderr + + def __str__(self): + message = "" + if self.stderr: + message += self.stderr + if self.stdout: + message += self.stdout + if message: + message = ":\n" + message + return (CmdError.__str__(self) + + " with exit code %s" % self.exitcode + message) + +class Popen(subprocess.Popen): + defaults = { + "close_fds": True, + "preexec_fn": subprocess_setup, + "stdout": subprocess.PIPE, + "stderr": subprocess.STDOUT, + "stdin": subprocess.PIPE, + "shell": False, + } + + def __init__(self, *args, **kwargs): + options = dict(self.defaults) + options.update(kwargs) + subprocess.Popen.__init__(self, *args, **options) + +def _logged_communicate(pipe, log, input, extrafiles): + if pipe.stdin: + if input is not None: + pipe.stdin.write(input) + pipe.stdin.close() + + outdata, errdata = [], [] + rin = [] + + if pipe.stdout is not None: + bb.utils.nonblockingfd(pipe.stdout.fileno()) + rin.append(pipe.stdout) + if pipe.stderr is not None: + bb.utils.nonblockingfd(pipe.stderr.fileno()) + rin.append(pipe.stderr) + for fobj, _ in extrafiles: + bb.utils.nonblockingfd(fobj.fileno()) + rin.append(fobj) + + def readextras(selected): + for fobj, func in extrafiles: + if fobj in selected: + try: + data = fobj.read() + except IOError as err: + if err.errno == errno.EAGAIN or err.errno == errno.EWOULDBLOCK: + data = None + if data is not None: + func(data) + + try: + while pipe.poll() is None: + rlist = rin + try: + r,w,e = select.select (rlist, [], [], 1) + except OSError as e: + if e.errno != errno.EINTR: + raise + + if pipe.stdout in r: + data = pipe.stdout.read() + if data is not None: + outdata.append(data) + log.write(data) + + if pipe.stderr in r: + data = pipe.stderr.read() + if data is not None: + errdata.append(data) + log.write(data) + + readextras(r) + + finally: + log.flush() + + readextras([fobj for fobj, _ in extrafiles]) + + if pipe.stdout is not None: + pipe.stdout.close() + if pipe.stderr is not None: + pipe.stderr.close() + return ''.join(outdata), ''.join(errdata) + +def run(cmd, input=None, log=None, extrafiles=None, **options): + """Convenience function to run a command and return its output, raising an + exception when the command fails""" + + if not extrafiles: + extrafiles = [] + + if isinstance(cmd, basestring) and not "shell" in options: + options["shell"] = True + + try: + pipe = Popen(cmd, **options) + except OSError as exc: + if exc.errno == 2: + raise NotFoundError(cmd) + else: + raise CmdError(cmd, exc) + + if log: + stdout, stderr = _logged_communicate(pipe, log, input, extrafiles) + else: + stdout, stderr = pipe.communicate(input) + + if pipe.returncode != 0: + raise ExecutionError(cmd, pipe.returncode, stdout, stderr) + return stdout, stderr diff --git a/import-layers/yocto-poky/bitbake/lib/bb/providers.py b/import-layers/yocto-poky/bitbake/lib/bb/providers.py new file mode 100644 index 000000000..563a091fd --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/providers.py @@ -0,0 +1,428 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH +# Copyright (C) 2006 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import re +import logging +from bb import data, utils +from collections import defaultdict +import bb + +logger = logging.getLogger("BitBake.Provider") + +class NoProvider(bb.BBHandledException): + """Exception raised when no provider of a build dependency can be found""" + +class NoRProvider(bb.BBHandledException): + """Exception raised when no provider of a runtime dependency can be found""" + +class MultipleRProvider(bb.BBHandledException): + """Exception raised when multiple providers of a runtime dependency can be found""" + +def findProviders(cfgData, dataCache, pkg_pn = None): + """ + Convenience function to get latest and preferred providers in pkg_pn + """ + + if not pkg_pn: + pkg_pn = dataCache.pkg_pn + + # Need to ensure data store is expanded + localdata = data.createCopy(cfgData) + bb.data.update_data(localdata) + bb.data.expandKeys(localdata) + + preferred_versions = {} + latest_versions = {} + + for pn in pkg_pn: + (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn) + preferred_versions[pn] = (pref_ver, pref_file) + latest_versions[pn] = (last_ver, last_file) + + return (latest_versions, preferred_versions) + + +def allProviders(dataCache): + """ + Find all providers for each pn + """ + all_providers = defaultdict(list) + for (fn, pn) in dataCache.pkg_fn.items(): + ver = dataCache.pkg_pepvpr[fn] + all_providers[pn].append((ver, fn)) + return all_providers + + +def sortPriorities(pn, dataCache, pkg_pn = None): + """ + Reorder pkg_pn by file priority and default preference + """ + + if not pkg_pn: + pkg_pn = dataCache.pkg_pn + + files = pkg_pn[pn] + priorities = {} + for f in files: + priority = dataCache.bbfile_priority[f] + preference = dataCache.pkg_dp[f] + if priority not in priorities: + priorities[priority] = {} + if preference not in priorities[priority]: + priorities[priority][preference] = [] + priorities[priority][preference].append(f) + tmp_pn = [] + for pri in sorted(priorities): + tmp_pref = [] + for pref in sorted(priorities[pri]): + tmp_pref.extend(priorities[pri][pref]) + tmp_pn = [tmp_pref] + tmp_pn + + return tmp_pn + +def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r): + """ + Check if the version pe,pv,pr is the preferred one. + If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%' + """ + if (pr == preferred_r or preferred_r == None): + if (pe == preferred_e or preferred_e == None): + if preferred_v == pv: + return True + if preferred_v != None and preferred_v.endswith('%') and pv.startswith(preferred_v[:len(preferred_v)-1]): + return True + return False + +def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): + """ + Find the first provider in pkg_pn with a PREFERRED_VERSION set. + """ + + preferred_file = None + preferred_ver = None + + # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot + # hence we do this manually rather than use OVERRIDES + preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn, True) + if not preferred_v: + preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn, True) + if not preferred_v: + preferred_v = cfgData.getVar("PREFERRED_VERSION", True) + + if preferred_v: + m = re.match('(\d+:)*(.*)(_.*)*', preferred_v) + if m: + if m.group(1): + preferred_e = m.group(1)[:-1] + else: + preferred_e = None + preferred_v = m.group(2) + if m.group(3): + preferred_r = m.group(3)[1:] + else: + preferred_r = None + else: + preferred_e = None + preferred_r = None + + for file_set in pkg_pn: + for f in file_set: + pe, pv, pr = dataCache.pkg_pepvpr[f] + if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r): + preferred_file = f + preferred_ver = (pe, pv, pr) + break + if preferred_file: + break; + if preferred_r: + pv_str = '%s-%s' % (preferred_v, preferred_r) + else: + pv_str = preferred_v + if not (preferred_e is None): + pv_str = '%s:%s' % (preferred_e, pv_str) + itemstr = "" + if item: + itemstr = " (for item %s)" % item + if preferred_file is None: + logger.info("preferred version %s of %s not available%s", pv_str, pn, itemstr) + available_vers = [] + for file_set in pkg_pn: + for f in file_set: + pe, pv, pr = dataCache.pkg_pepvpr[f] + ver_str = pv + if pe: + ver_str = "%s:%s" % (pe, ver_str) + if not ver_str in available_vers: + available_vers.append(ver_str) + if available_vers: + available_vers.sort() + logger.info("versions of %s available: %s", pn, ' '.join(available_vers)) + else: + logger.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr) + + return (preferred_ver, preferred_file) + + +def findLatestProvider(pn, cfgData, dataCache, file_set): + """ + Return the highest version of the providers in file_set. + Take default preferences into account. + """ + latest = None + latest_p = 0 + latest_f = None + for file_name in file_set: + pe, pv, pr = dataCache.pkg_pepvpr[file_name] + dp = dataCache.pkg_dp[file_name] + + if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p): + latest = (pe, pv, pr) + latest_f = file_name + latest_p = dp + + return (latest, latest_f) + + +def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): + """ + If there is a PREFERRED_VERSION, find the highest-priority bbfile + providing that version. If not, find the latest version provided by + an bbfile in the highest-priority set. + """ + + sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn) + # Find the highest priority provider with a PREFERRED_VERSION set + (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item) + # Find the latest version of the highest priority provider + (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0]) + + if preferred_file is None: + preferred_file = latest_f + preferred_ver = latest + + return (latest, latest_f, preferred_ver, preferred_file) + + +def _filterProviders(providers, item, cfgData, dataCache): + """ + Take a list of providers and filter/reorder according to the + environment variables + """ + eligible = [] + preferred_versions = {} + sortpkg_pn = {} + + # The order of providers depends on the order of the files on the disk + # up to here. Sort pkg_pn to make dependency issues reproducible rather + # than effectively random. + providers.sort() + + # Collate providers by PN + pkg_pn = {} + for p in providers: + pn = dataCache.pkg_fn[p] + if pn not in pkg_pn: + pkg_pn[pn] = [] + pkg_pn[pn].append(p) + + logger.debug(1, "providers for %s are: %s", item, pkg_pn.keys()) + + # First add PREFERRED_VERSIONS + for pn in pkg_pn: + sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn) + preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item) + if preferred_versions[pn][1]: + eligible.append(preferred_versions[pn][1]) + + # Now add latest versions + for pn in sortpkg_pn: + if pn in preferred_versions and preferred_versions[pn][1]: + continue + preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0]) + eligible.append(preferred_versions[pn][1]) + + if len(eligible) == 0: + logger.error("no eligible providers for %s", item) + return 0 + + # If pn == item, give it a slight default preference + # This means PREFERRED_PROVIDER_foobar defaults to foobar if available + for p in providers: + pn = dataCache.pkg_fn[p] + if pn != item: + continue + (newvers, fn) = preferred_versions[pn] + if not fn in eligible: + continue + eligible.remove(fn) + eligible = [fn] + eligible + + return eligible + + +def filterProviders(providers, item, cfgData, dataCache): + """ + Take a list of providers and filter/reorder according to the + environment variables + Takes a "normal" target item + """ + + eligible = _filterProviders(providers, item, cfgData, dataCache) + + prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item, True) + if prefervar: + dataCache.preferred[item] = prefervar + + foundUnique = False + if item in dataCache.preferred: + for p in eligible: + pn = dataCache.pkg_fn[p] + if dataCache.preferred[item] == pn: + logger.verbose("selecting %s to satisfy %s due to PREFERRED_PROVIDERS", pn, item) + eligible.remove(p) + eligible = [p] + eligible + foundUnique = True + break + + logger.debug(1, "sorted providers for %s are: %s", item, eligible) + + return eligible, foundUnique + +def filterProvidersRunTime(providers, item, cfgData, dataCache): + """ + Take a list of providers and filter/reorder according to the + environment variables + Takes a "runtime" target item + """ + + eligible = _filterProviders(providers, item, cfgData, dataCache) + + # First try and match any PREFERRED_RPROVIDER entry + prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item, True) + foundUnique = False + if prefervar: + for p in eligible: + pn = dataCache.pkg_fn[p] + if prefervar == pn: + logger.verbose("selecting %s to satisfy %s due to PREFERRED_RPROVIDER", pn, item) + eligible.remove(p) + eligible = [p] + eligible + foundUnique = True + numberPreferred = 1 + break + + # If we didn't find an RPROVIDER entry, try and infer the provider from PREFERRED_PROVIDER entries + # by looking through the provides of each eligible recipe and seeing if a PREFERRED_PROVIDER was set. + # This is most useful for virtual/ entries rather than having a RPROVIDER per entry. + if not foundUnique: + # Should use dataCache.preferred here? + preferred = [] + preferred_vars = [] + pns = {} + for p in eligible: + pns[dataCache.pkg_fn[p]] = p + for p in eligible: + pn = dataCache.pkg_fn[p] + provides = dataCache.pn_provides[pn] + for provide in provides: + prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide, True) + #logger.debug(1, "checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys()) + if prefervar in pns and pns[prefervar] not in preferred: + var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar) + logger.verbose("selecting %s to satisfy runtime %s due to %s", prefervar, item, var) + preferred_vars.append(var) + pref = pns[prefervar] + eligible.remove(pref) + eligible = [pref] + eligible + preferred.append(pref) + break + + numberPreferred = len(preferred) + + if numberPreferred > 1: + logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s. You could set PREFERRED_RPROVIDER_%s" % (item, preferred, preferred_vars, item)) + + logger.debug(1, "sorted runtime providers for %s are: %s", item, eligible) + + return eligible, numberPreferred + +regexp_cache = {} + +def getRuntimeProviders(dataCache, rdepend): + """ + Return any providers of runtime dependency + """ + rproviders = [] + + if rdepend in dataCache.rproviders: + rproviders += dataCache.rproviders[rdepend] + + if rdepend in dataCache.packages: + rproviders += dataCache.packages[rdepend] + + if rproviders: + return rproviders + + # Only search dynamic packages if we can't find anything in other variables + for pattern in dataCache.packages_dynamic: + pattern = pattern.replace('+', "\+") + if pattern in regexp_cache: + regexp = regexp_cache[pattern] + else: + try: + regexp = re.compile(pattern) + except: + logger.error("Error parsing regular expression '%s'", pattern) + raise + regexp_cache[pattern] = regexp + if regexp.match(rdepend): + rproviders += dataCache.packages_dynamic[pattern] + logger.debug(1, "Assuming %s is a dynamic package, but it may not exist" % rdepend) + + return rproviders + + +def buildWorldTargetList(dataCache): + """ + Build package list for "bitbake world" + """ + if dataCache.world_target: + return + + logger.debug(1, "collating packages for \"world\"") + for f in dataCache.possible_world: + terminal = True + pn = dataCache.pkg_fn[f] + + for p in dataCache.pn_provides[pn]: + if p.startswith('virtual/'): + logger.debug(2, "World build skipping %s due to %s provider starting with virtual/", f, p) + terminal = False + break + for pf in dataCache.providers[p]: + if dataCache.pkg_fn[pf] != pn: + logger.debug(2, "World build skipping %s due to both us and %s providing %s", f, pf, p) + terminal = False + break + if terminal: + dataCache.world_target.add(pn) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/pysh/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/pysh/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/import-layers/yocto-poky/bitbake/lib/bb/pysh/builtin.py b/import-layers/yocto-poky/bitbake/lib/bb/pysh/builtin.py new file mode 100644 index 000000000..b748e4a4f --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/pysh/builtin.py @@ -0,0 +1,710 @@ +# builtin.py - builtins and utilities definitions for pysh. +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +"""Builtin and internal utilities implementations. + +- Beware not to use python interpreter environment as if it were the shell +environment. For instance, commands working directory must be explicitely handled +through env['PWD'] instead of relying on python working directory. +""" +import errno +import optparse +import os +import re +import subprocess +import sys +import time + +def has_subprocess_bug(): + return getattr(subprocess, 'list2cmdline') and \ + ( subprocess.list2cmdline(['']) == '' or \ + subprocess.list2cmdline(['foo|bar']) == 'foo|bar') + +# Detect python bug 1634343: "subprocess swallows empty arguments under win32" +# <http://sourceforge.net/tracker/index.php?func=detail&aid=1634343&group_id=5470&atid=105470> +# Also detect: "[ 1710802 ] subprocess must escape redirection characters under win32" +# <http://sourceforge.net/tracker/index.php?func=detail&aid=1710802&group_id=5470&atid=105470> +if has_subprocess_bug(): + import subprocess_fix + subprocess.list2cmdline = subprocess_fix.list2cmdline + +from sherrors import * + +class NonExitingParser(optparse.OptionParser): + """OptionParser default behaviour upon error is to print the error message and + exit. Raise a utility error instead. + """ + def error(self, msg): + raise UtilityError(msg) + +#------------------------------------------------------------------------------- +# set special builtin +#------------------------------------------------------------------------------- +OPT_SET = NonExitingParser(usage="set - set or unset options and positional parameters") +OPT_SET.add_option( '-f', action='store_true', dest='has_f', default=False, + help='The shell shall disable pathname expansion.') +OPT_SET.add_option('-e', action='store_true', dest='has_e', default=False, + help="""When this option is on, if a simple command fails for any of the \ + reasons listed in Consequences of Shell Errors or returns an exit status \ + value >0, and is not part of the compound list following a while, until, \ + or if keyword, and is not a part of an AND or OR list, and is not a \ + pipeline preceded by the ! reserved word, then the shell shall immediately \ + exit.""") +OPT_SET.add_option('-x', action='store_true', dest='has_x', default=False, + help="""The shell shall write to standard error a trace for each command \ + after it expands the command and before it executes it. It is unspecified \ + whether the command that turns tracing off is traced.""") + +def builtin_set(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_SET.parse_args(args) + env = interp.get_env() + + if option.has_f: + env.set_opt('-f') + if option.has_e: + env.set_opt('-e') + if option.has_x: + env.set_opt('-x') + return 0 + +#------------------------------------------------------------------------------- +# shift special builtin +#------------------------------------------------------------------------------- +def builtin_shift(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + params = interp.get_env().get_positional_args() + if args: + try: + n = int(args[0]) + if n > len(params): + raise ValueError() + except ValueError: + return 1 + else: + n = 1 + + params[:n] = [] + interp.get_env().set_positional_args(params) + return 0 + +#------------------------------------------------------------------------------- +# export special builtin +#------------------------------------------------------------------------------- +OPT_EXPORT = NonExitingParser(usage="set - set or unset options and positional parameters") +OPT_EXPORT.add_option('-p', action='store_true', dest='has_p', default=False) + +def builtin_export(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_EXPORT.parse_args(args) + if option.has_p: + raise NotImplementedError() + + for arg in args: + try: + name, value = arg.split('=', 1) + except ValueError: + name, value = arg, None + env = interp.get_env().export(name, value) + + return 0 + +#------------------------------------------------------------------------------- +# return special builtin +#------------------------------------------------------------------------------- +def builtin_return(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + res = 0 + if args: + try: + res = int(args[0]) + except ValueError: + res = 0 + if not 0<=res<=255: + res = 0 + + # BUG: should be last executed command exit code + raise ReturnSignal(res) + +#------------------------------------------------------------------------------- +# trap special builtin +#------------------------------------------------------------------------------- +def builtin_trap(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + if len(args) < 2: + stderr.write('trap: usage: trap [[arg] signal_spec ...]\n') + return 2 + + action = args[0] + for sig in args[1:]: + try: + env.traps[sig] = action + except Exception as e: + stderr.write('trap: %s\n' % str(e)) + return 0 + +#------------------------------------------------------------------------------- +# unset special builtin +#------------------------------------------------------------------------------- +OPT_UNSET = NonExitingParser("unset - unset values and attributes of variables and functions") +OPT_UNSET.add_option( '-f', action='store_true', dest='has_f', default=False) +OPT_UNSET.add_option( '-v', action='store_true', dest='has_v', default=False) + +def builtin_unset(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_UNSET.parse_args(args) + + status = 0 + env = interp.get_env() + for arg in args: + try: + if option.has_f: + env.remove_function(arg) + else: + del env[arg] + except KeyError: + pass + except VarAssignmentError: + status = 1 + + return status + +#------------------------------------------------------------------------------- +# wait special builtin +#------------------------------------------------------------------------------- +def builtin_wait(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + return interp.wait([int(arg) for arg in args]) + +#------------------------------------------------------------------------------- +# cat utility +#------------------------------------------------------------------------------- +def utility_cat(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + if not args: + args = ['-'] + + status = 0 + for arg in args: + if arg == '-': + data = stdin.read() + else: + path = os.path.join(env['PWD'], arg) + try: + f = file(path, 'rb') + try: + data = f.read() + finally: + f.close() + except IOError as e: + if e.errno != errno.ENOENT: + raise + status = 1 + continue + stdout.write(data) + stdout.flush() + return status + +#------------------------------------------------------------------------------- +# cd utility +#------------------------------------------------------------------------------- +OPT_CD = NonExitingParser("cd - change the working directory") + +def utility_cd(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_CD.parse_args(args) + env = interp.get_env() + + directory = None + printdir = False + if not args: + home = env.get('HOME') + if home: + # Unspecified, do nothing + return 0 + else: + directory = home + elif len(args)==1: + directory = args[0] + if directory=='-': + if 'OLDPWD' not in env: + raise UtilityError("OLDPWD not set") + printdir = True + directory = env['OLDPWD'] + else: + raise UtilityError("too many arguments") + + curpath = None + # Absolute directories will be handled correctly by the os.path.join call. + if not directory.startswith('.') and not directory.startswith('..'): + cdpaths = env.get('CDPATH', '.').split(';') + for cdpath in cdpaths: + p = os.path.join(cdpath, directory) + if os.path.isdir(p): + curpath = p + break + + if curpath is None: + curpath = directory + curpath = os.path.join(env['PWD'], directory) + + env['OLDPWD'] = env['PWD'] + env['PWD'] = curpath + if printdir: + stdout.write('%s\n' % curpath) + return 0 + +#------------------------------------------------------------------------------- +# colon utility +#------------------------------------------------------------------------------- +def utility_colon(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + return 0 + +#------------------------------------------------------------------------------- +# echo utility +#------------------------------------------------------------------------------- +def utility_echo(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + # Echo only takes arguments, no options. Use printf if you need fancy stuff. + output = ' '.join(args) + '\n' + stdout.write(output) + stdout.flush() + return 0 + +#------------------------------------------------------------------------------- +# egrep utility +#------------------------------------------------------------------------------- +# egrep is usually a shell script. +# Unfortunately, pysh does not support shell scripts *with arguments* right now, +# so the redirection is implemented here, assuming grep is available. +def utility_egrep(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + return run_command('grep', ['-E'] + args, interp, env, stdin, stdout, + stderr, debugflags) + +#------------------------------------------------------------------------------- +# env utility +#------------------------------------------------------------------------------- +def utility_env(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + if args and args[0]=='-i': + raise NotImplementedError('env: -i option is not implemented') + + i = 0 + for arg in args: + if '=' not in arg: + break + # Update the current environment + name, value = arg.split('=', 1) + env[name] = value + i += 1 + + if args[i:]: + # Find then execute the specified interpreter + utility = env.find_in_path(args[i]) + if not utility: + return 127 + args[i:i+1] = utility + name = args[i] + args = args[i+1:] + try: + return run_command(name, args, interp, env, stdin, stdout, stderr, + debugflags) + except UtilityError: + stderr.write('env: failed to execute %s' % ' '.join([name]+args)) + return 126 + else: + for pair in env.get_variables().iteritems(): + stdout.write('%s=%s\n' % pair) + return 0 + +#------------------------------------------------------------------------------- +# exit utility +#------------------------------------------------------------------------------- +def utility_exit(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + res = None + if args: + try: + res = int(args[0]) + except ValueError: + res = None + if not 0<=res<=255: + res = None + + if res is None: + # BUG: should be last executed command exit code + res = 0 + + raise ExitSignal(res) + +#------------------------------------------------------------------------------- +# fgrep utility +#------------------------------------------------------------------------------- +# see egrep +def utility_fgrep(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + return run_command('grep', ['-F'] + args, interp, env, stdin, stdout, + stderr, debugflags) + +#------------------------------------------------------------------------------- +# gunzip utility +#------------------------------------------------------------------------------- +# see egrep +def utility_gunzip(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + return run_command('gzip', ['-d'] + args, interp, env, stdin, stdout, + stderr, debugflags) + +#------------------------------------------------------------------------------- +# kill utility +#------------------------------------------------------------------------------- +def utility_kill(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + for arg in args: + pid = int(arg) + status = subprocess.call(['pskill', '/T', str(pid)], + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # pskill is asynchronous, hence the stupid polling loop + while 1: + p = subprocess.Popen(['pslist', str(pid)], + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = p.communicate()[0] + if ('process %d was not' % pid) in output: + break + time.sleep(1) + return status + +#------------------------------------------------------------------------------- +# mkdir utility +#------------------------------------------------------------------------------- +OPT_MKDIR = NonExitingParser("mkdir - make directories.") +OPT_MKDIR.add_option('-p', action='store_true', dest='has_p', default=False) + +def utility_mkdir(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + # TODO: implement umask + # TODO: implement proper utility error report + option, args = OPT_MKDIR.parse_args(args) + for arg in args: + path = os.path.join(env['PWD'], arg) + if option.has_p: + try: + os.makedirs(path) + except IOError as e: + if e.errno != errno.EEXIST: + raise + else: + os.mkdir(path) + return 0 + +#------------------------------------------------------------------------------- +# netstat utility +#------------------------------------------------------------------------------- +def utility_netstat(name, args, interp, env, stdin, stdout, stderr, debugflags): + # Do you really expect me to implement netstat ? + # This empty form is enough for Mercurial tests since it's + # supposed to generate nothing upon success. Faking this test + # is not a big deal either. + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + return 0 + +#------------------------------------------------------------------------------- +# pwd utility +#------------------------------------------------------------------------------- +OPT_PWD = NonExitingParser("pwd - return working directory name") +OPT_PWD.add_option('-L', action='store_true', dest='has_L', default=True, + help="""If the PWD environment variable contains an absolute pathname of \ + the current directory that does not contain the filenames dot or dot-dot, \ + pwd shall write this pathname to standard output. Otherwise, the -L option \ + shall behave as the -P option.""") +OPT_PWD.add_option('-P', action='store_true', dest='has_L', default=False, + help="""The absolute pathname written shall not contain filenames that, in \ + the context of the pathname, refer to files of type symbolic link.""") + +def utility_pwd(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_PWD.parse_args(args) + stdout.write('%s\n' % env['PWD']) + return 0 + +#------------------------------------------------------------------------------- +# printf utility +#------------------------------------------------------------------------------- +RE_UNESCAPE = re.compile(r'(\\x[a-zA-Z0-9]{2}|\\[0-7]{1,3}|\\.)') + +def utility_printf(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + def replace(m): + assert m.group() + g = m.group()[1:] + if g.startswith('x'): + return chr(int(g[1:], 16)) + if len(g) <= 3 and len([c for c in g if c in '01234567']) == len(g): + # Yay, an octal number + return chr(int(g, 8)) + return { + 'a': '\a', + 'b': '\b', + 'f': '\f', + 'n': '\n', + 'r': '\r', + 't': '\t', + 'v': '\v', + '\\': '\\', + }.get(g) + + # Convert escape sequences + format = re.sub(RE_UNESCAPE, replace, args[0]) + stdout.write(format % tuple(args[1:])) + return 0 + +#------------------------------------------------------------------------------- +# true utility +#------------------------------------------------------------------------------- +def utility_true(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + return 0 + +#------------------------------------------------------------------------------- +# sed utility +#------------------------------------------------------------------------------- +RE_SED = re.compile(r'^s(.).*\1[a-zA-Z]*$') + +# cygwin sed fails with some expressions when they do not end with a single space. +# see unit tests for details. Interestingly, the same expressions works perfectly +# in cygwin shell. +def utility_sed(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + # Scan pattern arguments and append a space if necessary + for i in xrange(len(args)): + if not RE_SED.search(args[i]): + continue + args[i] = args[i] + ' ' + + return run_command(name, args, interp, env, stdin, stdout, + stderr, debugflags) + +#------------------------------------------------------------------------------- +# sleep utility +#------------------------------------------------------------------------------- +def utility_sleep(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + time.sleep(int(args[0])) + return 0 + +#------------------------------------------------------------------------------- +# sort utility +#------------------------------------------------------------------------------- +OPT_SORT = NonExitingParser("sort - sort, merge, or sequence check text files") + +def utility_sort(name, args, interp, env, stdin, stdout, stderr, debugflags): + + def sort(path): + if path == '-': + lines = stdin.readlines() + else: + try: + f = file(path) + try: + lines = f.readlines() + finally: + f.close() + except IOError as e: + stderr.write(str(e) + '\n') + return 1 + + if lines and lines[-1][-1]!='\n': + lines[-1] = lines[-1] + '\n' + return lines + + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_SORT.parse_args(args) + alllines = [] + + if len(args)<=0: + args += ['-'] + + # Load all files lines + curdir = os.getcwd() + try: + os.chdir(env['PWD']) + for path in args: + alllines += sort(path) + finally: + os.chdir(curdir) + + alllines.sort() + for line in alllines: + stdout.write(line) + return 0 + +#------------------------------------------------------------------------------- +# hg utility +#------------------------------------------------------------------------------- + +hgcommands = [ + 'add', + 'addremove', + 'commit', 'ci', + 'debugrename', + 'debugwalk', + 'falabala', # Dummy command used in a mercurial test + 'incoming', + 'locate', + 'pull', + 'push', + 'qinit', + 'remove', 'rm', + 'rename', 'mv', + 'revert', + 'showconfig', + 'status', 'st', + 'strip', + ] + +def rewriteslashes(name, args): + # Several hg commands output file paths, rewrite the separators + if len(args) > 1 and name.lower().endswith('python') \ + and args[0].endswith('hg'): + for cmd in hgcommands: + if cmd in args[1:]: + return True + + # svn output contains many paths with OS specific separators. + # Normalize these to unix paths. + base = os.path.basename(name) + if base.startswith('svn'): + return True + + return False + +def rewritehg(output): + if not output: + return output + # Rewrite os specific messages + output = output.replace(': The system cannot find the file specified', + ': No such file or directory') + output = re.sub(': Access is denied.*$', ': Permission denied', output) + output = output.replace(': No connection could be made because the target machine actively refused it', + ': Connection refused') + return output + + +def run_command(name, args, interp, env, stdin, stdout, + stderr, debugflags): + # Execute the command + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + hgbin = interp.options().hgbinary + ishg = hgbin and ('hg' in name or args and 'hg' in args[0]) + unixoutput = 'cygwin' in name or ishg + + exec_env = env.get_variables() + try: + # BUG: comparing file descriptor is clearly not a reliable way to tell + # whether they point on the same underlying object. But in pysh limited + # scope this is usually right, we do not expect complicated redirections + # besides usual 2>&1. + # Still there is one case we have but cannot deal with is when stdout + # and stderr are redirected *by pysh caller*. This the reason for the + # --redirect pysh() option. + # Now, we want to know they are the same because we sometimes need to + # transform the command output, mostly remove CR-LF to ensure that + # command output is unix-like. Cygwin utilies are a special case because + # they explicitely set their output streams to binary mode, so we have + # nothing to do. For all others commands, we have to guess whether they + # are sending text data, in which case the transformation must be done. + # Again, the NUL character test is unreliable but should be enough for + # hg tests. + redirected = stdout.fileno()==stderr.fileno() + if not redirected: + p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env, + stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env, + stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + out, err = p.communicate() + except WindowsError as e: + raise UtilityError(str(e)) + + if not unixoutput: + def encode(s): + if '\0' in s: + return s + return s.replace('\r\n', '\n') + else: + encode = lambda s: s + + if rewriteslashes(name, args): + encode1_ = encode + def encode(s): + s = encode1_(s) + s = s.replace('\\\\', '\\') + s = s.replace('\\', '/') + return s + + if ishg: + encode2_ = encode + def encode(s): + return rewritehg(encode2_(s)) + + stdout.write(encode(out)) + if not redirected: + stderr.write(encode(err)) + return p.returncode + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/pysh/interp.py b/import-layers/yocto-poky/bitbake/lib/bb/pysh/interp.py new file mode 100644 index 000000000..25d8c92ec --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/pysh/interp.py @@ -0,0 +1,1367 @@ +# interp.py - shell interpreter for pysh. +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +"""Implement the shell interpreter. + +Most references are made to "The Open Group Base Specifications Issue 6". +<http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html> +""" +# TODO: document the fact input streams must implement fileno() so Popen will work correctly. +# it requires non-stdin stream to be implemented as files. Still to be tested... +# DOC: pathsep is used in PATH instead of ':'. Clearly, there are path syntax issues here. +# TODO: stop command execution upon error. +# TODO: sort out the filename/io_number mess. It should be possible to use filenames only. +# TODO: review subshell implementation +# TODO: test environment cloning for non-special builtins +# TODO: set -x should not rebuild commands from tokens, assignments/redirections are lost +# TODO: unit test for variable assignment +# TODO: test error management wrt error type/utility type +# TODO: test for binary output everywhere +# BUG: debug-parsing does not pass log file to PLY. Maybe a PLY upgrade is necessary. +import base64 +import cPickle as pickle +import errno +import glob +import os +import re +import subprocess +import sys +import tempfile + +try: + s = set() + del s +except NameError: + from Set import Set as set + +import builtin +from sherrors import * +import pyshlex +import pyshyacc + +def mappend(func, *args, **kargs): + """Like map but assume func returns a list. Returned lists are merged into + a single one. + """ + return reduce(lambda a,b: a+b, map(func, *args, **kargs), []) + +class FileWrapper: + """File object wrapper to ease debugging. + + Allow mode checking and implement file duplication through a simple + reference counting scheme. Not sure the latter is really useful since + only real file descriptors can be used. + """ + def __init__(self, mode, file, close=True): + if mode not in ('r', 'w', 'a'): + raise IOError('invalid mode: %s' % mode) + self._mode = mode + self._close = close + if isinstance(file, FileWrapper): + if file._refcount[0] <= 0: + raise IOError(0, 'Error') + self._refcount = file._refcount + self._refcount[0] += 1 + self._file = file._file + else: + self._refcount = [1] + self._file = file + + def dup(self): + return FileWrapper(self._mode, self, self._close) + + def fileno(self): + """fileno() should be only necessary for input streams.""" + return self._file.fileno() + + def read(self, size=-1): + if self._mode!='r': + raise IOError(0, 'Error') + return self._file.read(size) + + def readlines(self, *args, **kwargs): + return self._file.readlines(*args, **kwargs) + + def write(self, s): + if self._mode not in ('w', 'a'): + raise IOError(0, 'Error') + return self._file.write(s) + + def flush(self): + self._file.flush() + + def close(self): + if not self._refcount: + return + assert self._refcount[0] > 0 + + self._refcount[0] -= 1 + if self._refcount[0] == 0: + self._mode = 'c' + if self._close: + self._file.close() + self._refcount = None + + def mode(self): + return self._mode + + def __getattr__(self, name): + if name == 'name': + self.name = getattr(self._file, name) + return self.name + else: + raise AttributeError(name) + + def __del__(self): + self.close() + + +def win32_open_devnull(mode): + return open('NUL', mode) + + +class Redirections: + """Stores open files and their mapping to pseudo-sh file descriptor. + """ + # BUG: redirections are not handled correctly: 1>&3 2>&3 3>&4 does + # not make 1 to redirect to 4 + def __init__(self, stdin=None, stdout=None, stderr=None): + self._descriptors = {} + if stdin is not None: + self._add_descriptor(0, stdin) + if stdout is not None: + self._add_descriptor(1, stdout) + if stderr is not None: + self._add_descriptor(2, stderr) + + def add_here_document(self, interp, name, content, io_number=None): + if io_number is None: + io_number = 0 + + if name==pyshlex.unquote_wordtree(name): + content = interp.expand_here_document(('TOKEN', content)) + + # Write document content in a temporary file + tmp = tempfile.TemporaryFile() + try: + tmp.write(content) + tmp.flush() + tmp.seek(0) + self._add_descriptor(io_number, FileWrapper('r', tmp)) + except: + tmp.close() + raise + + def add(self, interp, op, filename, io_number=None): + if op not in ('<', '>', '>|', '>>', '>&'): + # TODO: add descriptor duplication and here_documents + raise RedirectionError('Unsupported redirection operator "%s"' % op) + + if io_number is not None: + io_number = int(io_number) + + if (op == '>&' and filename.isdigit()) or filename=='-': + # No expansion for file descriptors, quote them if you want a filename + fullname = filename + else: + if filename.startswith('/'): + # TODO: win32 kludge + if filename=='/dev/null': + fullname = 'NUL' + else: + # TODO: handle absolute pathnames, they are unlikely to exist on the + # current platform (win32 for instance). + raise NotImplementedError() + else: + fullname = interp.expand_redirection(('TOKEN', filename)) + if not fullname: + raise RedirectionError('%s: ambiguous redirect' % filename) + # Build absolute path based on PWD + fullname = os.path.join(interp.get_env()['PWD'], fullname) + + if op=='<': + return self._add_input_redirection(interp, fullname, io_number) + elif op in ('>', '>|'): + clobber = ('>|'==op) + return self._add_output_redirection(interp, fullname, io_number, clobber) + elif op=='>>': + return self._add_output_appending(interp, fullname, io_number) + elif op=='>&': + return self._dup_output_descriptor(fullname, io_number) + + def close(self): + if self._descriptors is not None: + for desc in self._descriptors.itervalues(): + desc.flush() + desc.close() + self._descriptors = None + + def stdin(self): + return self._descriptors[0] + + def stdout(self): + return self._descriptors[1] + + def stderr(self): + return self._descriptors[2] + + def clone(self): + clone = Redirections() + for desc, fileobj in self._descriptors.iteritems(): + clone._descriptors[desc] = fileobj.dup() + return clone + + def _add_output_redirection(self, interp, filename, io_number, clobber): + if io_number is None: + # io_number default to standard output + io_number = 1 + + if not clobber and interp.get_env().has_opt('-C') and os.path.isfile(filename): + # File already exist in no-clobber mode, bail out + raise RedirectionError('File "%s" already exists' % filename) + + # Open and register + self._add_file_descriptor(io_number, filename, 'w') + + def _add_output_appending(self, interp, filename, io_number): + if io_number is None: + io_number = 1 + self._add_file_descriptor(io_number, filename, 'a') + + def _add_input_redirection(self, interp, filename, io_number): + if io_number is None: + io_number = 0 + self._add_file_descriptor(io_number, filename, 'r') + + def _add_file_descriptor(self, io_number, filename, mode): + try: + if filename.startswith('/'): + if filename=='/dev/null': + f = win32_open_devnull(mode+'b') + else: + # TODO: handle absolute pathnames, they are unlikely to exist on the + # current platform (win32 for instance). + raise NotImplementedError('cannot open absolute path %s' % repr(filename)) + else: + f = file(filename, mode+'b') + except IOError as e: + raise RedirectionError(str(e)) + + wrapper = None + try: + wrapper = FileWrapper(mode, f) + f = None + self._add_descriptor(io_number, wrapper) + except: + if f: f.close() + if wrapper: wrapper.close() + raise + + def _dup_output_descriptor(self, source_fd, dest_fd): + if source_fd is None: + source_fd = 1 + self._dup_file_descriptor(source_fd, dest_fd, 'w') + + def _dup_file_descriptor(self, source_fd, dest_fd, mode): + source_fd = int(source_fd) + if source_fd not in self._descriptors: + raise RedirectionError('"%s" is not a valid file descriptor' % str(source_fd)) + source = self._descriptors[source_fd] + + if source.mode()!=mode: + raise RedirectionError('Descriptor %s cannot be duplicated in mode "%s"' % (str(source), mode)) + + if dest_fd=='-': + # Close the source descriptor + del self._descriptors[source_fd] + source.close() + else: + dest_fd = int(dest_fd) + if dest_fd not in self._descriptors: + raise RedirectionError('Cannot replace file descriptor %s' % str(dest_fd)) + + dest = self._descriptors[dest_fd] + if dest.mode()!=mode: + raise RedirectionError('Descriptor %s cannot be cannot be redirected in mode "%s"' % (str(dest), mode)) + + self._descriptors[dest_fd] = source.dup() + dest.close() + + def _add_descriptor(self, io_number, file): + io_number = int(io_number) + + if io_number in self._descriptors: + # Close the current descriptor + d = self._descriptors[io_number] + del self._descriptors[io_number] + d.close() + + self._descriptors[io_number] = file + + def __str__(self): + names = [('%d=%r' % (k, getattr(v, 'name', None))) for k,v + in self._descriptors.iteritems()] + names = ','.join(names) + return 'Redirections(%s)' % names + + def __del__(self): + self.close() + +def cygwin_to_windows_path(path): + """Turn /cygdrive/c/foo into c:/foo, or return path if it + is not a cygwin path. + """ + if not path.startswith('/cygdrive/'): + return path + path = path[len('/cygdrive/'):] + path = path[:1] + ':' + path[1:] + return path + +def win32_to_unix_path(path): + if path is not None: + path = path.replace('\\', '/') + return path + +_RE_SHEBANG = re.compile(r'^\#!\s?([^\s]+)(?:\s([^\s]+))?') +_SHEBANG_CMDS = { + '/usr/bin/env': 'env', + '/bin/sh': 'pysh', + 'python': 'python', +} + +def resolve_shebang(path, ignoreshell=False): + """Return a list of arguments as shebang interpreter call or an empty list + if path does not refer to an executable script. + See <http://www.opengroup.org/austin/docs/austin_51r2.txt>. + + ignoreshell - set to True to ignore sh shebangs. Return an empty list instead. + """ + try: + f = file(path) + try: + # At most 80 characters in the first line + header = f.read(80).splitlines()[0] + finally: + f.close() + + m = _RE_SHEBANG.search(header) + if not m: + return [] + cmd, arg = m.group(1,2) + if os.path.isfile(cmd): + # Keep this one, the hg script for instance contains a weird windows + # shebang referencing the current python install. + cmdfile = os.path.basename(cmd).lower() + if cmdfile == 'python.exe': + cmd = 'python' + pass + elif cmd not in _SHEBANG_CMDS: + raise CommandNotFound('Unknown interpreter "%s" referenced in '\ + 'shebang' % header) + cmd = _SHEBANG_CMDS.get(cmd) + if cmd is None or (ignoreshell and cmd == 'pysh'): + return [] + if arg is None: + return [cmd, win32_to_unix_path(path)] + return [cmd, arg, win32_to_unix_path(path)] + except IOError as e: + if e.errno!=errno.ENOENT and \ + (e.errno!=errno.EPERM and not os.path.isdir(path)): # Opening a directory raises EPERM + raise + return [] + +def win32_find_in_path(name, path): + if isinstance(path, str): + path = path.split(os.pathsep) + + exts = os.environ.get('PATHEXT', '').lower().split(os.pathsep) + for p in path: + p_name = os.path.join(p, name) + + prefix = resolve_shebang(p_name) + if prefix: + return prefix + + for ext in exts: + p_name_ext = p_name + ext + if os.path.exists(p_name_ext): + return [win32_to_unix_path(p_name_ext)] + return [] + +class Traps(dict): + def __setitem__(self, key, value): + if key not in ('EXIT',): + raise NotImplementedError() + super(Traps, self).__setitem__(key, value) + +# IFS white spaces character class +_IFS_WHITESPACES = (' ', '\t', '\n') + +class Environment: + """Environment holds environment variables, export table, function + definitions and whatever is defined in 2.12 "Shell Execution Environment", + redirection excepted. + """ + def __init__(self, pwd): + self._opt = set() #Shell options + + self._functions = {} + self._env = {'?': '0', '#': '0'} + self._exported = set([ + 'HOME', 'IFS', 'PATH' + ]) + + # Set environment vars with side-effects + self._ifs_ws = None # Set of IFS whitespace characters + self._ifs_re = None # Regular expression used to split between words using IFS classes + self['IFS'] = ''.join(_IFS_WHITESPACES) #Default environment values + self['PWD'] = pwd + self.traps = Traps() + + def clone(self, subshell=False): + env = Environment(self['PWD']) + env._opt = set(self._opt) + for k,v in self.get_variables().iteritems(): + if k in self._exported: + env.export(k,v) + elif subshell: + env[k] = v + + if subshell: + env._functions = dict(self._functions) + + return env + + def __getitem__(self, key): + if key in ('@', '*', '-', '$'): + raise NotImplementedError('%s is not implemented' % repr(key)) + return self._env[key] + + def get(self, key, defval=None): + try: + return self[key] + except KeyError: + return defval + + def __setitem__(self, key, value): + if key=='IFS': + # Update the whitespace/non-whitespace classes + self._update_ifs(value) + elif key=='PWD': + pwd = os.path.abspath(value) + if not os.path.isdir(pwd): + raise VarAssignmentError('Invalid directory %s' % value) + value = pwd + elif key in ('?', '!'): + value = str(int(value)) + self._env[key] = value + + def __delitem__(self, key): + if key in ('IFS', 'PWD', '?'): + raise VarAssignmentError('%s cannot be unset' % key) + del self._env[key] + + def __contains__(self, item): + return item in self._env + + def set_positional_args(self, args): + """Set the content of 'args' as positional argument from 1 to len(args). + Return previous argument as a list of strings. + """ + # Save and remove previous arguments + prevargs = [] + for i in xrange(int(self._env['#'])): + i = str(i+1) + prevargs.append(self._env[i]) + del self._env[i] + self._env['#'] = '0' + + #Set new ones + for i,arg in enumerate(args): + self._env[str(i+1)] = str(arg) + self._env['#'] = str(len(args)) + + return prevargs + + def get_positional_args(self): + return [self._env[str(i+1)] for i in xrange(int(self._env['#']))] + + def get_variables(self): + return dict(self._env) + + def export(self, key, value=None): + if value is not None: + self[key] = value + self._exported.add(key) + + def get_exported(self): + return [(k,self._env.get(k)) for k in self._exported] + + def split_fields(self, word): + if not self._ifs_ws or not word: + return [word] + return re.split(self._ifs_re, word) + + def _update_ifs(self, value): + """Update the split_fields related variables when IFS character set is + changed. + """ + # TODO: handle NULL IFS + + # Separate characters in whitespace and non-whitespace + chars = set(value) + ws = [c for c in chars if c in _IFS_WHITESPACES] + nws = [c for c in chars if c not in _IFS_WHITESPACES] + + # Keep whitespaces in a string for left and right stripping + self._ifs_ws = ''.join(ws) + + # Build a regexp to split fields + trailing = '[' + ''.join([re.escape(c) for c in ws]) + ']' + if nws: + # First, the single non-whitespace occurence. + nws = '[' + ''.join([re.escape(c) for c in nws]) + ']' + nws = '(?:' + trailing + '*' + nws + trailing + '*' + '|' + trailing + '+)' + else: + # Then mix all parts with quantifiers + nws = trailing + '+' + self._ifs_re = re.compile(nws) + + def has_opt(self, opt, val=None): + return (opt, val) in self._opt + + def set_opt(self, opt, val=None): + self._opt.add((opt, val)) + + def find_in_path(self, name, pwd=False): + path = self._env.get('PATH', '').split(os.pathsep) + if pwd: + path[:0] = [self['PWD']] + if os.name == 'nt': + return win32_find_in_path(name, self._env.get('PATH', '')) + else: + raise NotImplementedError() + + def define_function(self, name, body): + if not is_name(name): + raise ShellSyntaxError('%s is not a valid function name' % repr(name)) + self._functions[name] = body + + def remove_function(self, name): + del self._functions[name] + + def is_function(self, name): + return name in self._functions + + def get_function(self, name): + return self._functions.get(name) + + +name_charset = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_' +name_charset = dict(zip(name_charset,name_charset)) + +def match_name(s): + """Return the length in characters of the longest prefix made of name + allowed characters in s. + """ + for i,c in enumerate(s): + if c not in name_charset: + return s[:i] + return s + +def is_name(s): + return len([c for c in s if c not in name_charset])<=0 + +def is_special_param(c): + return len(c)==1 and c in ('@','*','#','?','-','$','!','0') + +def utility_not_implemented(name, *args, **kwargs): + raise NotImplementedError('%s utility is not implemented' % name) + + +class Utility: + """Define utilities properties: + func -- utility callable. See builtin module for utility samples. + is_special -- see XCU 2.8. + """ + def __init__(self, func, is_special=0): + self.func = func + self.is_special = bool(is_special) + + +def encodeargs(args): + def encodearg(s): + lines = base64.encodestring(s) + lines = [l.splitlines()[0] for l in lines] + return ''.join(lines) + + s = pickle.dumps(args) + return encodearg(s) + +def decodeargs(s): + s = base64.decodestring(s) + return pickle.loads(s) + + +class GlobError(Exception): + pass + +class Options: + def __init__(self): + # True if Mercurial operates with binary streams + self.hgbinary = True + +class Interpreter: + # Implementation is very basic: the execute() method just makes a DFS on the + # AST and execute nodes one by one. Nodes are tuple (name,obj) where name + # is a string identifier and obj the AST element returned by the parser. + # + # Handler are named after the node identifiers. + # TODO: check node names and remove the switch in execute with some + # dynamic getattr() call to find node handlers. + """Shell interpreter. + + The following debugging flags can be passed: + debug-parsing - enable PLY debugging. + debug-tree - print the generated AST. + debug-cmd - trace command execution before word expansion, plus exit status. + debug-utility - trace utility execution. + """ + + # List supported commands. + COMMANDS = { + 'cat': Utility(builtin.utility_cat,), + 'cd': Utility(builtin.utility_cd,), + ':': Utility(builtin.utility_colon,), + 'echo': Utility(builtin.utility_echo), + 'env': Utility(builtin.utility_env), + 'exit': Utility(builtin.utility_exit), + 'export': Utility(builtin.builtin_export, is_special=1), + 'egrep': Utility(builtin.utility_egrep), + 'fgrep': Utility(builtin.utility_fgrep), + 'gunzip': Utility(builtin.utility_gunzip), + 'kill': Utility(builtin.utility_kill), + 'mkdir': Utility(builtin.utility_mkdir), + 'netstat': Utility(builtin.utility_netstat), + 'printf': Utility(builtin.utility_printf), + 'pwd': Utility(builtin.utility_pwd), + 'return': Utility(builtin.builtin_return, is_special=1), + 'sed': Utility(builtin.utility_sed,), + 'set': Utility(builtin.builtin_set,), + 'shift': Utility(builtin.builtin_shift,), + 'sleep': Utility(builtin.utility_sleep,), + 'sort': Utility(builtin.utility_sort,), + 'trap': Utility(builtin.builtin_trap, is_special=1), + 'true': Utility(builtin.utility_true), + 'unset': Utility(builtin.builtin_unset, is_special=1), + 'wait': Utility(builtin.builtin_wait, is_special=1), + } + + def __init__(self, pwd, debugflags = [], env=None, redirs=None, stdin=None, + stdout=None, stderr=None, opts=Options()): + self._env = env + if self._env is None: + self._env = Environment(pwd) + self._children = {} + + self._redirs = redirs + self._close_redirs = False + + if self._redirs is None: + if stdin is None: + stdin = sys.stdin + if stdout is None: + stdout = sys.stdout + if stderr is None: + stderr = sys.stderr + stdin = FileWrapper('r', stdin, False) + stdout = FileWrapper('w', stdout, False) + stderr = FileWrapper('w', stderr, False) + self._redirs = Redirections(stdin, stdout, stderr) + self._close_redirs = True + + self._debugflags = list(debugflags) + self._logfile = sys.stderr + self._options = opts + + def close(self): + """Must be called when the interpreter is no longer used.""" + script = self._env.traps.get('EXIT') + if script: + try: + self.execute_script(script=script) + except: + pass + + if self._redirs is not None and self._close_redirs: + self._redirs.close() + self._redirs = None + + def log(self, s): + self._logfile.write(s) + self._logfile.flush() + + def __getitem__(self, key): + return self._env[key] + + def __setitem__(self, key, value): + self._env[key] = value + + def options(self): + return self._options + + def redirect(self, redirs, ios): + def add_redir(io): + if isinstance(io, pyshyacc.IORedirect): + redirs.add(self, io.op, io.filename, io.io_number) + else: + redirs.add_here_document(self, io.name, io.content, io.io_number) + + map(add_redir, ios) + return redirs + + def execute_script(self, script=None, ast=None, sourced=False, + scriptpath=None): + """If script is not None, parse the input. Otherwise takes the supplied + AST. Then execute the AST. + Return the script exit status. + """ + try: + if scriptpath is not None: + self._env['0'] = os.path.abspath(scriptpath) + + if script is not None: + debug_parsing = ('debug-parsing' in self._debugflags) + cmds, script = pyshyacc.parse(script, True, debug_parsing) + if 'debug-tree' in self._debugflags: + pyshyacc.print_commands(cmds, self._logfile) + self._logfile.flush() + else: + cmds, script = ast, '' + + status = 0 + for cmd in cmds: + try: + status = self.execute(cmd) + except ExitSignal as e: + if sourced: + raise + status = int(e.args[0]) + return status + except ShellError: + self._env['?'] = 1 + raise + if 'debug-utility' in self._debugflags or 'debug-cmd' in self._debugflags: + self.log('returncode ' + str(status)+ '\n') + return status + except CommandNotFound as e: + print >>self._redirs.stderr, str(e) + self._redirs.stderr.flush() + # Command not found by non-interactive shell + # return 127 + raise + except RedirectionError as e: + # TODO: should be handled depending on the utility status + print >>self._redirs.stderr, str(e) + self._redirs.stderr.flush() + # Command not found by non-interactive shell + # return 127 + raise + + def dotcommand(self, env, args): + if len(args) < 1: + raise ShellError('. expects at least one argument') + path = args[0] + if '/' not in path: + found = env.find_in_path(args[0], True) + if found: + path = found[0] + script = file(path).read() + return self.execute_script(script=script, sourced=True) + + def execute(self, token, redirs=None): + """Execute and AST subtree with supplied redirections overriding default + interpreter ones. + Return the exit status. + """ + if not token: + return 0 + + if redirs is None: + redirs = self._redirs + + if isinstance(token, list): + # Commands sequence + res = 0 + for t in token: + res = self.execute(t, redirs) + return res + + type, value = token + status = 0 + if type=='simple_command': + redirs_copy = redirs.clone() + try: + # TODO: define and handle command return values + # TODO: implement set -e + status = self._execute_simple_command(value, redirs_copy) + finally: + redirs_copy.close() + elif type=='pipeline': + status = self._execute_pipeline(value, redirs) + elif type=='and_or': + status = self._execute_and_or(value, redirs) + elif type=='for_clause': + status = self._execute_for_clause(value, redirs) + elif type=='while_clause': + status = self._execute_while_clause(value, redirs) + elif type=='function_definition': + status = self._execute_function_definition(value, redirs) + elif type=='brace_group': + status = self._execute_brace_group(value, redirs) + elif type=='if_clause': + status = self._execute_if_clause(value, redirs) + elif type=='subshell': + status = self.subshell(ast=value.cmds, redirs=redirs) + elif type=='async': + status = self._asynclist(value) + elif type=='redirect_list': + redirs_copy = self.redirect(redirs.clone(), value.redirs) + try: + status = self.execute(value.cmd, redirs_copy) + finally: + redirs_copy.close() + else: + raise NotImplementedError('Unsupported token type ' + type) + + if status < 0: + status = 255 + return status + + def _execute_if_clause(self, if_clause, redirs): + cond_status = self.execute(if_clause.cond, redirs) + if cond_status==0: + return self.execute(if_clause.if_cmds, redirs) + else: + return self.execute(if_clause.else_cmds, redirs) + + def _execute_brace_group(self, group, redirs): + status = 0 + for cmd in group.cmds: + status = self.execute(cmd, redirs) + return status + + def _execute_function_definition(self, fundef, redirs): + self._env.define_function(fundef.name, fundef.body) + return 0 + + def _execute_while_clause(self, while_clause, redirs): + status = 0 + while 1: + cond_status = 0 + for cond in while_clause.condition: + cond_status = self.execute(cond, redirs) + + if cond_status: + break + + for cmd in while_clause.cmds: + status = self.execute(cmd, redirs) + + return status + + def _execute_for_clause(self, for_clause, redirs): + if not is_name(for_clause.name): + raise ShellSyntaxError('%s is not a valid name' % repr(for_clause.name)) + items = mappend(self.expand_token, for_clause.items) + + status = 0 + for item in items: + self._env[for_clause.name] = item + for cmd in for_clause.cmds: + status = self.execute(cmd, redirs) + return status + + def _execute_and_or(self, or_and, redirs): + res = self.execute(or_and.left, redirs) + if (or_and.op=='&&' and res==0) or (or_and.op!='&&' and res!=0): + res = self.execute(or_and.right, redirs) + return res + + def _execute_pipeline(self, pipeline, redirs): + if len(pipeline.commands)==1: + status = self.execute(pipeline.commands[0], redirs) + else: + # Execute all commands one after the other + status = 0 + inpath, outpath = None, None + try: + # Commands inputs and outputs cannot really be plugged as done + # by a real shell. Run commands sequentially and chain their + # input/output throught temporary files. + tmpfd, inpath = tempfile.mkstemp() + os.close(tmpfd) + tmpfd, outpath = tempfile.mkstemp() + os.close(tmpfd) + + inpath = win32_to_unix_path(inpath) + outpath = win32_to_unix_path(outpath) + + for i, cmd in enumerate(pipeline.commands): + call_redirs = redirs.clone() + try: + if i!=0: + call_redirs.add(self, '<', inpath) + if i!=len(pipeline.commands)-1: + call_redirs.add(self, '>', outpath) + + status = self.execute(cmd, call_redirs) + + # Chain inputs/outputs + inpath, outpath = outpath, inpath + finally: + call_redirs.close() + finally: + if inpath: os.remove(inpath) + if outpath: os.remove(outpath) + + if pipeline.reverse_status: + status = int(not status) + self._env['?'] = status + return status + + def _execute_function(self, name, args, interp, env, stdin, stdout, stderr, *others): + assert interp is self + + func = env.get_function(name) + #Set positional parameters + prevargs = None + try: + prevargs = env.set_positional_args(args) + try: + redirs = Redirections(stdin.dup(), stdout.dup(), stderr.dup()) + try: + status = self.execute(func, redirs) + finally: + redirs.close() + except ReturnSignal as e: + status = int(e.args[0]) + env['?'] = status + return status + finally: + #Reset positional parameters + if prevargs is not None: + env.set_positional_args(prevargs) + + def _execute_simple_command(self, token, redirs): + """Can raise ReturnSignal when return builtin is called, ExitSignal when + exit is called, and other shell exceptions upon builtin failures. + """ + debug_command = 'debug-cmd' in self._debugflags + if debug_command: + self.log('word' + repr(token.words) + '\n') + self.log('assigns' + repr(token.assigns) + '\n') + self.log('redirs' + repr(token.redirs) + '\n') + + is_special = None + env = self._env + + try: + # Word expansion + args = [] + for word in token.words: + args += self.expand_token(word) + if is_special is None and args: + is_special = env.is_function(args[0]) or \ + (args[0] in self.COMMANDS and self.COMMANDS[args[0]].is_special) + + if debug_command: + self.log('_execute_simple_command' + str(args) + '\n') + + if not args: + # Redirections happen is a subshell + redirs = redirs.clone() + elif not is_special: + env = self._env.clone() + + # Redirections + self.redirect(redirs, token.redirs) + + # Variables assignments + res = 0 + for type,(k,v) in token.assigns: + status, expanded = self.expand_variable((k,v)) + if status is not None: + res = status + if args: + env.export(k, expanded) + else: + env[k] = expanded + + if args and args[0] in ('.', 'source'): + res = self.dotcommand(env, args[1:]) + elif args: + if args[0] in self.COMMANDS: + command = self.COMMANDS[args[0]] + elif env.is_function(args[0]): + command = Utility(self._execute_function, is_special=True) + else: + if not '/' in args[0].replace('\\', '/'): + cmd = env.find_in_path(args[0]) + if not cmd: + # TODO: test error code on unknown command => 127 + raise CommandNotFound('Unknown command: "%s"' % args[0]) + else: + # Handle commands like '/cygdrive/c/foo.bat' + cmd = cygwin_to_windows_path(args[0]) + if not os.path.exists(cmd): + raise CommandNotFound('%s: No such file or directory' % args[0]) + shebang = resolve_shebang(cmd) + if shebang: + cmd = shebang + else: + cmd = [cmd] + args[0:1] = cmd + command = Utility(builtin.run_command) + + # Command execution + if 'debug-cmd' in self._debugflags: + self.log('redirections ' + str(redirs) + '\n') + + res = command.func(args[0], args[1:], self, env, + redirs.stdin(), redirs.stdout(), + redirs.stderr(), self._debugflags) + + if self._env.has_opt('-x'): + # Trace command execution in shell environment + # BUG: would be hard to reproduce a real shell behaviour since + # the AST is not annotated with source lines/tokens. + self._redirs.stdout().write(' '.join(args)) + + except ReturnSignal: + raise + except ShellError as e: + if is_special or isinstance(e, (ExitSignal, + ShellSyntaxError, ExpansionError)): + raise e + self._redirs.stderr().write(str(e)+'\n') + return 1 + + return res + + def expand_token(self, word): + """Expand a word as specified in [2.6 Word Expansions]. Return the list + of expanded words. + """ + status, wtrees = self._expand_word(word) + return map(pyshlex.wordtree_as_string, wtrees) + + def expand_variable(self, word): + """Return a status code (or None if no command expansion occurred) + and a single word. + """ + status, wtrees = self._expand_word(word, pathname=False, split=False) + words = map(pyshlex.wordtree_as_string, wtrees) + assert len(words)==1 + return status, words[0] + + def expand_here_document(self, word): + """Return the expanded document as a single word. The here document is + assumed to be unquoted. + """ + status, wtrees = self._expand_word(word, pathname=False, + split=False, here_document=True) + words = map(pyshlex.wordtree_as_string, wtrees) + assert len(words)==1 + return words[0] + + def expand_redirection(self, word): + """Return a single word.""" + return self.expand_variable(word)[1] + + def get_env(self): + return self._env + + def _expand_word(self, token, pathname=True, split=True, here_document=False): + wtree = pyshlex.make_wordtree(token[1], here_document=here_document) + + # TODO: implement tilde expansion + def expand(wtree): + """Return a pseudo wordtree: the tree or its subelements can be empty + lists when no value result from the expansion. + """ + status = None + for part in wtree: + if not isinstance(part, list): + continue + if part[0]in ("'", '\\'): + continue + elif part[0] in ('`', '$('): + status, result = self._expand_command(part) + part[:] = result + elif part[0] in ('$', '${'): + part[:] = self._expand_parameter(part, wtree[0]=='"', split) + elif part[0] in ('', '"'): + status, result = expand(part) + part[:] = result + else: + raise NotImplementedError('%s expansion is not implemented' + % part[0]) + # [] is returned when an expansion result in no-field, + # like an empty $@ + wtree = [p for p in wtree if p != []] + if len(wtree) < 3: + return status, [] + return status, wtree + + status, wtree = expand(wtree) + if len(wtree) == 0: + return status, wtree + wtree = pyshlex.normalize_wordtree(wtree) + + if split: + wtrees = self._split_fields(wtree) + else: + wtrees = [wtree] + + if pathname: + wtrees = mappend(self._expand_pathname, wtrees) + + wtrees = map(self._remove_quotes, wtrees) + return status, wtrees + + def _expand_command(self, wtree): + # BUG: there is something to do with backslashes and quoted + # characters here + command = pyshlex.wordtree_as_string(wtree[1:-1]) + status, output = self.subshell_output(command) + return status, ['', output, ''] + + def _expand_parameter(self, wtree, quoted=False, split=False): + """Return a valid wtree or an empty list when no parameter results.""" + # Get the parameter name + # TODO: implement weird expansion rules with ':' + name = pyshlex.wordtree_as_string(wtree[1:-1]) + if not is_name(name) and not is_special_param(name): + raise ExpansionError('Bad substitution "%s"' % name) + # TODO: implement special parameters + if name in ('@', '*'): + args = self._env.get_positional_args() + if len(args) == 0: + return [] + if len(args)<2: + return ['', ''.join(args), ''] + + sep = self._env.get('IFS', '')[:1] + if split and quoted and name=='@': + # Introduce a new token to tell the caller that these parameters + # cause a split as specified in 2.5.2 + return ['@'] + args + [''] + else: + return ['', sep.join(args), ''] + + return ['', self._env.get(name, ''), ''] + + def _split_fields(self, wtree): + def is_empty(split): + return split==['', '', ''] + + def split_positional(quoted): + # Return a list of wtree split according positional parameters rules. + # All remaining '@' groups are removed. + assert quoted[0]=='"' + + splits = [[]] + for part in quoted: + if not isinstance(part, list) or part[0]!='@': + splits[-1].append(part) + else: + # Empty or single argument list were dealt with already + assert len(part)>3 + # First argument must join with the beginning part of the original word + splits[-1].append(part[1]) + # Create double-quotes expressions for every argument after the first + for arg in part[2:-1]: + splits[-1].append('"') + splits.append(['"', arg]) + return splits + + # At this point, all expansions but pathnames have occured. Only quoted + # and positional sequences remain. Thus, all candidates for field splitting + # are in the tree root, or are positional splits ('@') and lie in root + # children. + if not wtree or wtree[0] not in ('', '"'): + # The whole token is quoted or empty, nothing to split + return [wtree] + + if wtree[0]=='"': + wtree = ['', wtree, ''] + + result = [['', '']] + for part in wtree[1:-1]: + if isinstance(part, list): + if part[0]=='"': + splits = split_positional(part) + if len(splits)<=1: + result[-1] += [part, ''] + else: + # Terminate the current split + result[-1] += [splits[0], ''] + result += splits[1:-1] + # Create a new split + result += [['', splits[-1], '']] + else: + result[-1] += [part, ''] + else: + splits = self._env.split_fields(part) + if len(splits)<=1: + # No split + result[-1][-1] += part + else: + # Terminate the current resulting part and create a new one + result[-1][-1] += splits[0] + result[-1].append('') + result += [['', r, ''] for r in splits[1:-1]] + result += [['', splits[-1]]] + result[-1].append('') + + # Leading and trailing empty groups come from leading/trailing blanks + if result and is_empty(result[-1]): + result[-1:] = [] + if result and is_empty(result[0]): + result[:1] = [] + return result + + def _expand_pathname(self, wtree): + """See [2.6.6 Pathname Expansion].""" + if self._env.has_opt('-f'): + return [wtree] + + # All expansions have been performed, only quoted sequences should remain + # in the tree. Generate the pattern by folding the tree, escaping special + # characters when appear quoted + special_chars = '*?[]' + + def make_pattern(wtree): + subpattern = [] + for part in wtree[1:-1]: + if isinstance(part, list): + part = make_pattern(part) + elif wtree[0]!='': + for c in part: + # Meta-characters cannot be quoted + if c in special_chars: + raise GlobError() + subpattern.append(part) + return ''.join(subpattern) + + def pwd_glob(pattern): + cwd = os.getcwd() + os.chdir(self._env['PWD']) + try: + return glob.glob(pattern) + finally: + os.chdir(cwd) + + #TODO: check working directory issues here wrt relative patterns + try: + pattern = make_pattern(wtree) + paths = pwd_glob(pattern) + except GlobError: + # BUG: Meta-characters were found in quoted sequences. The should + # have been used literally but this is unsupported in current glob module. + # Instead we consider the whole tree must be used literally and + # therefore there is no point in globbing. This is wrong when meta + # characters are mixed with quoted meta in the same pattern like: + # < foo*"py*" > + paths = [] + + if not paths: + return [wtree] + return [['', path, ''] for path in paths] + + def _remove_quotes(self, wtree): + """See [2.6.7 Quote Removal].""" + + def unquote(wtree): + unquoted = [] + for part in wtree[1:-1]: + if isinstance(part, list): + part = unquote(part) + unquoted.append(part) + return ''.join(unquoted) + + return ['', unquote(wtree), ''] + + def subshell(self, script=None, ast=None, redirs=None): + """Execute the script or AST in a subshell, with inherited redirections + if redirs is not None. + """ + if redirs: + sub_redirs = redirs + else: + sub_redirs = redirs.clone() + + subshell = None + try: + subshell = Interpreter(None, self._debugflags, self._env.clone(True), + sub_redirs, opts=self._options) + return subshell.execute_script(script, ast) + finally: + if not redirs: sub_redirs.close() + if subshell: subshell.close() + + def subshell_output(self, script): + """Execute the script in a subshell and return the captured output.""" + # Create temporary file to capture subshell output + tmpfd, tmppath = tempfile.mkstemp() + try: + tmpfile = os.fdopen(tmpfd, 'wb') + stdout = FileWrapper('w', tmpfile) + + redirs = Redirections(self._redirs.stdin().dup(), + stdout, + self._redirs.stderr().dup()) + try: + status = self.subshell(script=script, redirs=redirs) + finally: + redirs.close() + redirs = None + + # Extract subshell standard output + tmpfile = open(tmppath, 'rb') + try: + output = tmpfile.read() + return status, output.rstrip('\n') + finally: + tmpfile.close() + finally: + os.remove(tmppath) + + def _asynclist(self, cmd): + args = (self._env.get_variables(), cmd) + arg = encodeargs(args) + assert len(args) < 30*1024 + cmd = ['pysh.bat', '--ast', '-c', arg] + p = subprocess.Popen(cmd, cwd=self._env['PWD']) + self._children[p.pid] = p + self._env['!'] = p.pid + return 0 + + def wait(self, pids=None): + if not pids: + pids = self._children.keys() + + status = 127 + for pid in pids: + if pid not in self._children: + continue + p = self._children.pop(pid) + status = p.wait() + + return status + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/pysh/lsprof.py b/import-layers/yocto-poky/bitbake/lib/bb/pysh/lsprof.py new file mode 100644 index 000000000..b1831c22a --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/pysh/lsprof.py @@ -0,0 +1,116 @@ +#! /usr/bin/env python + +import sys +from _lsprof import Profiler, profiler_entry + +__all__ = ['profile', 'Stats'] + +def profile(f, *args, **kwds): + """XXX docstring""" + p = Profiler() + p.enable(subcalls=True, builtins=True) + try: + f(*args, **kwds) + finally: + p.disable() + return Stats(p.getstats()) + + +class Stats(object): + """XXX docstring""" + + def __init__(self, data): + self.data = data + + def sort(self, crit="inlinetime"): + """XXX docstring""" + if crit not in profiler_entry.__dict__: + raise ValueError("Can't sort by %s" % crit) + self.data.sort(lambda b, a: cmp(getattr(a, crit), + getattr(b, crit))) + for e in self.data: + if e.calls: + e.calls.sort(lambda b, a: cmp(getattr(a, crit), + getattr(b, crit))) + + def pprint(self, top=None, file=None, limit=None, climit=None): + """XXX docstring""" + if file is None: + file = sys.stdout + d = self.data + if top is not None: + d = d[:top] + cols = "% 12s %12s %11.4f %11.4f %s\n" + hcols = "% 12s %12s %12s %12s %s\n" + cols2 = "+%12s %12s %11.4f %11.4f + %s\n" + file.write(hcols % ("CallCount", "Recursive", "Total(ms)", + "Inline(ms)", "module:lineno(function)")) + count = 0 + for e in d: + file.write(cols % (e.callcount, e.reccallcount, e.totaltime, + e.inlinetime, label(e.code))) + count += 1 + if limit is not None and count == limit: + return + ccount = 0 + if e.calls: + for se in e.calls: + file.write(cols % ("+%s" % se.callcount, se.reccallcount, + se.totaltime, se.inlinetime, + "+%s" % label(se.code))) + count += 1 + ccount += 1 + if limit is not None and count == limit: + return + if climit is not None and ccount == climit: + break + + def freeze(self): + """Replace all references to code objects with string + descriptions; this makes it possible to pickle the instance.""" + + # this code is probably rather ickier than it needs to be! + for i in range(len(self.data)): + e = self.data[i] + if not isinstance(e.code, str): + self.data[i] = type(e)((label(e.code),) + e[1:]) + if e.calls: + for j in range(len(e.calls)): + se = e.calls[j] + if not isinstance(se.code, str): + e.calls[j] = type(se)((label(se.code),) + se[1:]) + +_fn2mod = {} + +def label(code): + if isinstance(code, str): + return code + try: + mname = _fn2mod[code.co_filename] + except KeyError: + for k, v in sys.modules.items(): + if v is None: + continue + if not hasattr(v, '__file__'): + continue + if not isinstance(v.__file__, str): + continue + if v.__file__.startswith(code.co_filename): + mname = _fn2mod[code.co_filename] = k + break + else: + mname = _fn2mod[code.co_filename] = '<%s>'%code.co_filename + + return '%s:%d(%s)' % (mname, code.co_firstlineno, code.co_name) + + +if __name__ == '__main__': + import os + sys.argv = sys.argv[1:] + if not sys.argv: + print >> sys.stderr, "usage: lsprof.py <script> <arguments...>" + sys.exit(2) + sys.path.insert(0, os.path.abspath(os.path.dirname(sys.argv[0]))) + stats = profile(execfile, sys.argv[0], globals(), locals()) + stats.sort() + stats.pprint() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/pysh/pysh.py b/import-layers/yocto-poky/bitbake/lib/bb/pysh/pysh.py new file mode 100644 index 000000000..b4e6145b5 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/pysh/pysh.py @@ -0,0 +1,167 @@ +# pysh.py - command processing for pysh. +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +import optparse +import os +import sys + +import interp + +SH_OPT = optparse.OptionParser(prog='pysh', usage="%prog [OPTIONS]", version='0.1') +SH_OPT.add_option('-c', action='store_true', dest='command_string', default=None, + help='A string that shall be interpreted by the shell as one or more commands') +SH_OPT.add_option('--redirect-to', dest='redirect_to', default=None, + help='Redirect script commands stdout and stderr to the specified file') +# See utility_command in builtin.py about the reason for this flag. +SH_OPT.add_option('--redirected', dest='redirected', action='store_true', default=False, + help='Tell the interpreter that stdout and stderr are actually the same objects, which is really stdout') +SH_OPT.add_option('--debug-parsing', action='store_true', dest='debug_parsing', default=False, + help='Trace PLY execution') +SH_OPT.add_option('--debug-tree', action='store_true', dest='debug_tree', default=False, + help='Display the generated syntax tree.') +SH_OPT.add_option('--debug-cmd', action='store_true', dest='debug_cmd', default=False, + help='Trace command execution before parameters expansion and exit status.') +SH_OPT.add_option('--debug-utility', action='store_true', dest='debug_utility', default=False, + help='Trace utility calls, after parameters expansions') +SH_OPT.add_option('--ast', action='store_true', dest='ast', default=False, + help='Encoded commands to execute in a subprocess') +SH_OPT.add_option('--profile', action='store_true', default=False, + help='Profile pysh run') + + +def split_args(args): + # Separate shell arguments from command ones + # Just stop at the first argument not starting with a dash. I know, this is completely broken, + # it ignores files starting with a dash or may take option values for command file. This is not + # supposed to happen for now + command_index = len(args) + for i,arg in enumerate(args): + if not arg.startswith('-'): + command_index = i + break + + return args[:command_index], args[command_index:] + + +def fixenv(env): + path = env.get('PATH') + if path is not None: + parts = path.split(os.pathsep) + # Remove Windows utilities from PATH, they are useless at best and + # some of them (find) may be confused with other utilities. + parts = [p for p in parts if 'system32' not in p.lower()] + env['PATH'] = os.pathsep.join(parts) + if env.get('HOME') is None: + # Several utilities, including cvsps, cannot work without + # a defined HOME directory. + env['HOME'] = os.path.expanduser('~') + return env + +def _sh(cwd, shargs, cmdargs, options, debugflags=None, env=None): + if os.environ.get('PYSH_TEXT') != '1': + import msvcrt + for fp in (sys.stdin, sys.stdout, sys.stderr): + msvcrt.setmode(fp.fileno(), os.O_BINARY) + + hgbin = os.environ.get('PYSH_HGTEXT') != '1' + + if debugflags is None: + debugflags = [] + if options.debug_parsing: debugflags.append('debug-parsing') + if options.debug_utility: debugflags.append('debug-utility') + if options.debug_cmd: debugflags.append('debug-cmd') + if options.debug_tree: debugflags.append('debug-tree') + + if env is None: + env = fixenv(dict(os.environ)) + if cwd is None: + cwd = os.getcwd() + + if not cmdargs: + # Nothing to do + return 0 + + ast = None + command_file = None + if options.command_string: + input = cmdargs[0] + if not options.ast: + input += '\n' + else: + args, input = interp.decodeargs(input), None + env, ast = args + cwd = env.get('PWD', cwd) + else: + command_file = cmdargs[0] + arguments = cmdargs[1:] + + prefix = interp.resolve_shebang(command_file, ignoreshell=True) + if prefix: + input = ' '.join(prefix + [command_file] + arguments) + else: + # Read commands from file + f = file(command_file) + try: + # Trailing newline to help the parser + input = f.read() + '\n' + finally: + f.close() + + redirect = None + try: + if options.redirected: + stdout = sys.stdout + stderr = stdout + elif options.redirect_to: + redirect = open(options.redirect_to, 'wb') + stdout = redirect + stderr = redirect + else: + stdout = sys.stdout + stderr = sys.stderr + + # TODO: set arguments to environment variables + opts = interp.Options() + opts.hgbinary = hgbin + ip = interp.Interpreter(cwd, debugflags, stdout=stdout, stderr=stderr, + opts=opts) + try: + # Export given environment in shell object + for k,v in env.iteritems(): + ip.get_env().export(k,v) + return ip.execute_script(input, ast, scriptpath=command_file) + finally: + ip.close() + finally: + if redirect is not None: + redirect.close() + +def sh(cwd=None, args=None, debugflags=None, env=None): + if args is None: + args = sys.argv[1:] + shargs, cmdargs = split_args(args) + options, shargs = SH_OPT.parse_args(shargs) + + if options.profile: + import lsprof + p = lsprof.Profiler() + p.enable(subcalls=True) + try: + return _sh(cwd, shargs, cmdargs, options, debugflags, env) + finally: + p.disable() + stats = lsprof.Stats(p.getstats()) + stats.sort() + stats.pprint(top=10, file=sys.stderr, climit=5) + else: + return _sh(cwd, shargs, cmdargs, options, debugflags, env) + +def main(): + sys.exit(sh()) + +if __name__=='__main__': + main() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/pysh/pyshlex.py b/import-layers/yocto-poky/bitbake/lib/bb/pysh/pyshlex.py new file mode 100644 index 000000000..b30123675 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/pysh/pyshlex.py @@ -0,0 +1,888 @@ +# pyshlex.py - PLY compatible lexer for pysh. +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +# TODO: +# - review all "char in 'abc'" snippets: the empty string can be matched +# - test line continuations within quoted/expansion strings +# - eof is buggy wrt sublexers +# - the lexer cannot really work in pull mode as it would be required to run +# PLY in pull mode. It was designed to work incrementally and it would not be +# that hard to enable pull mode. +import re +try: + s = set() + del s +except NameError: + from Set import Set as set + +from ply import lex +from sherrors import * + +class NeedMore(Exception): + pass + +def is_blank(c): + return c in (' ', '\t') + +_RE_DIGITS = re.compile(r'^\d+$') + +def are_digits(s): + return _RE_DIGITS.search(s) is not None + +_OPERATORS = dict([ + ('&&', 'AND_IF'), + ('||', 'OR_IF'), + (';;', 'DSEMI'), + ('<<', 'DLESS'), + ('>>', 'DGREAT'), + ('<&', 'LESSAND'), + ('>&', 'GREATAND'), + ('<>', 'LESSGREAT'), + ('<<-', 'DLESSDASH'), + ('>|', 'CLOBBER'), + ('&', 'AMP'), + (';', 'COMMA'), + ('<', 'LESS'), + ('>', 'GREATER'), + ('(', 'LPARENS'), + (')', 'RPARENS'), +]) + +#Make a function to silence pychecker "Local variable shadows global" +def make_partial_ops(): + partials = {} + for k in _OPERATORS: + for i in range(1, len(k)+1): + partials[k[:i]] = None + return partials + +_PARTIAL_OPERATORS = make_partial_ops() + +def is_partial_op(s): + """Return True if s matches a non-empty subpart of an operator starting + at its first character. + """ + return s in _PARTIAL_OPERATORS + +def is_op(s): + """If s matches an operator, returns the operator identifier. Return None + otherwise. + """ + return _OPERATORS.get(s) + +_RESERVEDS = dict([ + ('if', 'If'), + ('then', 'Then'), + ('else', 'Else'), + ('elif', 'Elif'), + ('fi', 'Fi'), + ('do', 'Do'), + ('done', 'Done'), + ('case', 'Case'), + ('esac', 'Esac'), + ('while', 'While'), + ('until', 'Until'), + ('for', 'For'), + ('{', 'Lbrace'), + ('}', 'Rbrace'), + ('!', 'Bang'), + ('in', 'In'), + ('|', 'PIPE'), +]) + +def get_reserved(s): + return _RESERVEDS.get(s) + +_RE_NAME = re.compile(r'^[0-9a-zA-Z_]+$') + +def is_name(s): + return _RE_NAME.search(s) is not None + +def find_chars(seq, chars): + for i,v in enumerate(seq): + if v in chars: + return i,v + return -1, None + +class WordLexer: + """WordLexer parse quoted or expansion expressions and return an expression + tree. The input string can be any well formed sequence beginning with quoting + or expansion character. Embedded expressions are handled recursively. The + resulting tree is made of lists and strings. Lists represent quoted or + expansion expressions. Each list first element is the opening separator, + the last one the closing separator. In-between can be any number of strings + or lists for sub-expressions. Non quoted/expansion expression can written as + strings or as lists with empty strings as starting and ending delimiters. + """ + + NAME_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_' + NAME_CHARSET = dict(zip(NAME_CHARSET, NAME_CHARSET)) + + SPECIAL_CHARSET = '@*#?-$!0' + + #Characters which can be escaped depends on the current delimiters + ESCAPABLE = { + '`': set(['$', '\\', '`']), + '"': set(['$', '\\', '`', '"']), + "'": set(), + } + + def __init__(self, heredoc = False): + # _buffer is the unprocessed input characters buffer + self._buffer = [] + # _stack is empty or contains a quoted list being processed + # (this is the DFS path to the quoted expression being evaluated). + self._stack = [] + self._escapable = None + # True when parsing unquoted here documents + self._heredoc = heredoc + + def add(self, data, eof=False): + """Feed the lexer with more data. If the quoted expression can be + delimited, return a tuple (expr, remaining) containing the expression + tree and the unconsumed data. + Otherwise, raise NeedMore. + """ + self._buffer += list(data) + self._parse(eof) + + result = self._stack[0] + remaining = ''.join(self._buffer) + self._stack = [] + self._buffer = [] + return result, remaining + + def _is_escapable(self, c, delim=None): + if delim is None: + if self._heredoc: + # Backslashes works as if they were double quoted in unquoted + # here-documents + delim = '"' + else: + if len(self._stack)<=1: + return True + delim = self._stack[-2][0] + + escapables = self.ESCAPABLE.get(delim, None) + return escapables is None or c in escapables + + def _parse_squote(self, buf, result, eof): + if not buf: + raise NeedMore() + try: + pos = buf.index("'") + except ValueError: + raise NeedMore() + result[-1] += ''.join(buf[:pos]) + result += ["'"] + return pos+1, True + + def _parse_bquote(self, buf, result, eof): + if not buf: + raise NeedMore() + + if buf[0]=='\n': + #Remove line continuations + result[:] = ['', '', ''] + elif self._is_escapable(buf[0]): + result[-1] += buf[0] + result += [''] + else: + #Keep as such + result[:] = ['', '\\'+buf[0], ''] + + return 1, True + + def _parse_dquote(self, buf, result, eof): + if not buf: + raise NeedMore() + pos, sep = find_chars(buf, '$\\`"') + if pos==-1: + raise NeedMore() + + result[-1] += ''.join(buf[:pos]) + if sep=='"': + result += ['"'] + return pos+1, True + else: + #Keep everything until the separator and defer processing + return pos, False + + def _parse_command(self, buf, result, eof): + if not buf: + raise NeedMore() + + chars = '$\\`"\'' + if result[0] == '$(': + chars += ')' + pos, sep = find_chars(buf, chars) + if pos == -1: + raise NeedMore() + + result[-1] += ''.join(buf[:pos]) + if (result[0]=='$(' and sep==')') or (result[0]=='`' and sep=='`'): + result += [sep] + return pos+1, True + else: + return pos, False + + def _parse_parameter(self, buf, result, eof): + if not buf: + raise NeedMore() + + pos, sep = find_chars(buf, '$\\`"\'}') + if pos==-1: + raise NeedMore() + + result[-1] += ''.join(buf[:pos]) + if sep=='}': + result += [sep] + return pos+1, True + else: + return pos, False + + def _parse_dollar(self, buf, result, eof): + sep = result[0] + if sep=='$': + if not buf: + #TODO: handle empty $ + raise NeedMore() + if buf[0]=='(': + if len(buf)==1: + raise NeedMore() + + if buf[1]=='(': + result[0] = '$((' + buf[:2] = [] + else: + result[0] = '$(' + buf[:1] = [] + + elif buf[0]=='{': + result[0] = '${' + buf[:1] = [] + else: + if buf[0] in self.SPECIAL_CHARSET: + result[-1] = buf[0] + read = 1 + else: + for read,c in enumerate(buf): + if c not in self.NAME_CHARSET: + break + else: + if not eof: + raise NeedMore() + read += 1 + + result[-1] += ''.join(buf[0:read]) + + if not result[-1]: + result[:] = ['', result[0], ''] + else: + result += [''] + return read,True + + sep = result[0] + if sep=='$(': + parsefunc = self._parse_command + elif sep=='${': + parsefunc = self._parse_parameter + else: + raise NotImplementedError(sep) + + pos, closed = parsefunc(buf, result, eof) + return pos, closed + + def _parse(self, eof): + buf = self._buffer + stack = self._stack + recurse = False + + while 1: + if not stack or recurse: + if not buf: + raise NeedMore() + if buf[0] not in ('"\\`$\''): + raise ShellSyntaxError('Invalid quoted string sequence') + stack.append([buf[0], '']) + buf[:1] = [] + recurse = False + + result = stack[-1] + if result[0]=="'": + parsefunc = self._parse_squote + elif result[0]=='\\': + parsefunc = self._parse_bquote + elif result[0]=='"': + parsefunc = self._parse_dquote + elif result[0]=='`': + parsefunc = self._parse_command + elif result[0][0]=='$': + parsefunc = self._parse_dollar + else: + raise NotImplementedError() + + read, closed = parsefunc(buf, result, eof) + + buf[:read] = [] + if closed: + if len(stack)>1: + #Merge in parent expression + parsed = stack.pop() + stack[-1] += [parsed] + stack[-1] += [''] + else: + break + else: + recurse = True + +def normalize_wordtree(wtree): + """Fold back every literal sequence (delimited with empty strings) into + parent sequence. + """ + def normalize(wtree): + result = [] + for part in wtree[1:-1]: + if isinstance(part, list): + part = normalize(part) + if part[0]=='': + #Move the part content back at current level + result += part[1:-1] + continue + elif not part: + #Remove empty strings + continue + result.append(part) + if not result: + result = [''] + return [wtree[0]] + result + [wtree[-1]] + + return normalize(wtree) + + +def make_wordtree(token, here_document=False): + """Parse a delimited token and return a tree similar to the ones returned by + WordLexer. token may contain any combinations of expansion/quoted fields and + non-ones. + """ + tree = [''] + remaining = token + delimiters = '\\$`' + if not here_document: + delimiters += '\'"' + + while 1: + pos, sep = find_chars(remaining, delimiters) + if pos==-1: + tree += [remaining, ''] + return normalize_wordtree(tree) + tree.append(remaining[:pos]) + remaining = remaining[pos:] + + try: + result, remaining = WordLexer(heredoc = here_document).add(remaining, True) + except NeedMore: + raise ShellSyntaxError('Invalid token "%s"') + tree.append(result) + + +def wordtree_as_string(wtree): + """Rewrite an expression tree generated by make_wordtree as string.""" + def visit(node, output): + for child in node: + if isinstance(child, list): + visit(child, output) + else: + output.append(child) + + output = [] + visit(wtree, output) + return ''.join(output) + + +def unquote_wordtree(wtree): + """Fold the word tree while removing quotes everywhere. Other expansion + sequences are joined as such. + """ + def unquote(wtree): + unquoted = [] + if wtree[0] in ('', "'", '"', '\\'): + wtree = wtree[1:-1] + + for part in wtree: + if isinstance(part, list): + part = unquote(part) + unquoted.append(part) + return ''.join(unquoted) + + return unquote(wtree) + + +class HereDocLexer: + """HereDocLexer delimits whatever comes from the here-document starting newline + not included to the closing delimiter line included. + """ + def __init__(self, op, delim): + assert op in ('<<', '<<-') + if not delim: + raise ShellSyntaxError('invalid here document delimiter %s' % str(delim)) + + self._op = op + self._delim = delim + self._buffer = [] + self._token = [] + + def add(self, data, eof): + """If the here-document was delimited, return a tuple (content, remaining). + Raise NeedMore() otherwise. + """ + self._buffer += list(data) + self._parse(eof) + token = ''.join(self._token) + remaining = ''.join(self._buffer) + self._token, self._remaining = [], [] + return token, remaining + + def _parse(self, eof): + while 1: + #Look for first unescaped newline. Quotes may be ignored + escaped = False + for i,c in enumerate(self._buffer): + if escaped: + escaped = False + elif c=='\\': + escaped = True + elif c=='\n': + break + else: + i = -1 + + if i==-1 or self._buffer[i]!='\n': + if not eof: + raise NeedMore() + #No more data, maybe the last line is closing delimiter + line = ''.join(self._buffer) + eol = '' + self._buffer[:] = [] + else: + line = ''.join(self._buffer[:i]) + eol = self._buffer[i] + self._buffer[:i+1] = [] + + if self._op=='<<-': + line = line.lstrip('\t') + + if line==self._delim: + break + + self._token += [line, eol] + if i==-1: + break + +class Token: + #TODO: check this is still in use + OPERATOR = 'OPERATOR' + WORD = 'WORD' + + def __init__(self): + self.value = '' + self.type = None + + def __getitem__(self, key): + #Behave like a two elements tuple + if key==0: + return self.type + if key==1: + return self.value + raise IndexError(key) + + +class HereDoc: + def __init__(self, op, name=None): + self.op = op + self.name = name + self.pendings = [] + +TK_COMMA = 'COMMA' +TK_AMPERSAND = 'AMP' +TK_OP = 'OP' +TK_TOKEN = 'TOKEN' +TK_COMMENT = 'COMMENT' +TK_NEWLINE = 'NEWLINE' +TK_IONUMBER = 'IO_NUMBER' +TK_ASSIGNMENT = 'ASSIGNMENT_WORD' +TK_HERENAME = 'HERENAME' + +class Lexer: + """Main lexer. + + Call add() until the script AST is returned. + """ + # Here-document handling makes the whole thing more complex because they basically + # force tokens to be reordered: here-content must come right after the operator + # and the here-document name, while some other tokens might be following the + # here-document expression on the same line. + # + # So, here-doc states are basically: + # *self._state==ST_NORMAL + # - self._heredoc.op is None: no here-document + # - self._heredoc.op is not None but name is: here-document operator matched, + # waiting for the document name/delimiter + # - self._heredoc.op and name are not None: here-document is ready, following + # tokens are being stored and will be pushed again when the document is + # completely parsed. + # *self._state==ST_HEREDOC + # - The here-document is being delimited by self._herelexer. Once it is done + # the content is pushed in front of the pending token list then all these + # tokens are pushed once again. + ST_NORMAL = 'ST_NORMAL' + ST_OP = 'ST_OP' + ST_BACKSLASH = 'ST_BACKSLASH' + ST_QUOTED = 'ST_QUOTED' + ST_COMMENT = 'ST_COMMENT' + ST_HEREDOC = 'ST_HEREDOC' + + #Match end of backquote strings + RE_BACKQUOTE_END = re.compile(r'(?<!\\)(`)') + + def __init__(self, parent_state = None): + self._input = [] + self._pos = 0 + + self._token = '' + self._type = TK_TOKEN + + self._state = self.ST_NORMAL + self._parent_state = parent_state + self._wordlexer = None + + self._heredoc = HereDoc(None) + self._herelexer = None + + ### Following attributes are not used for delimiting token and can safely + ### be changed after here-document detection (see _push_toke) + + # Count the number of tokens following a 'For' reserved word. Needed to + # return an 'In' reserved word if it comes in third place. + self._for_count = None + + def add(self, data, eof=False): + """Feed the lexer with data. + + When eof is set to True, returns unconsumed data or raise if the lexer + is in the middle of a delimiting operation. + Raise NeedMore otherwise. + """ + self._input += list(data) + self._parse(eof) + self._input[:self._pos] = [] + return ''.join(self._input) + + def _parse(self, eof): + while self._state: + if self._pos>=len(self._input): + if not eof: + raise NeedMore() + elif self._state not in (self.ST_OP, self.ST_QUOTED, self.ST_HEREDOC): + #Delimit the current token and leave cleanly + self._push_token('') + break + else: + #Let the sublexer handle the eof themselves + pass + + if self._state==self.ST_NORMAL: + self._parse_normal() + elif self._state==self.ST_COMMENT: + self._parse_comment() + elif self._state==self.ST_OP: + self._parse_op(eof) + elif self._state==self.ST_QUOTED: + self._parse_quoted(eof) + elif self._state==self.ST_HEREDOC: + self._parse_heredoc(eof) + else: + assert False, "Unknown state " + str(self._state) + + if self._heredoc.op is not None: + raise ShellSyntaxError('missing here-document delimiter') + + def _parse_normal(self): + c = self._input[self._pos] + if c=='\n': + self._push_token(c) + self._token = c + self._type = TK_NEWLINE + self._push_token('') + self._pos += 1 + elif c in ('\\', '\'', '"', '`', '$'): + self._state = self.ST_QUOTED + elif is_partial_op(c): + self._push_token(c) + + self._type = TK_OP + self._token += c + self._pos += 1 + self._state = self.ST_OP + elif is_blank(c): + self._push_token(c) + + #Discard blanks + self._pos += 1 + elif self._token: + self._token += c + self._pos += 1 + elif c=='#': + self._state = self.ST_COMMENT + self._type = TK_COMMENT + self._pos += 1 + else: + self._pos += 1 + self._token += c + + def _parse_op(self, eof): + assert self._token + + while 1: + if self._pos>=len(self._input): + if not eof: + raise NeedMore() + c = '' + else: + c = self._input[self._pos] + + op = self._token + c + if c and is_partial_op(op): + #Still parsing an operator + self._token = op + self._pos += 1 + else: + #End of operator + self._push_token(c) + self._state = self.ST_NORMAL + break + + def _parse_comment(self): + while 1: + if self._pos>=len(self._input): + raise NeedMore() + + c = self._input[self._pos] + if c=='\n': + #End of comment, do not consume the end of line + self._state = self.ST_NORMAL + break + else: + self._token += c + self._pos += 1 + + def _parse_quoted(self, eof): + """Precondition: the starting backquote/dollar is still in the input queue.""" + if not self._wordlexer: + self._wordlexer = WordLexer() + + if self._pos<len(self._input): + #Transfer input queue character into the subparser + input = self._input[self._pos:] + self._pos += len(input) + + wtree, remaining = self._wordlexer.add(input, eof) + self._wordlexer = None + self._token += wordtree_as_string(wtree) + + #Put unparsed character back in the input queue + if remaining: + self._input[self._pos:self._pos] = list(remaining) + self._state = self.ST_NORMAL + + def _parse_heredoc(self, eof): + assert not self._token + + if self._herelexer is None: + self._herelexer = HereDocLexer(self._heredoc.op, self._heredoc.name) + + if self._pos<len(self._input): + #Transfer input queue character into the subparser + input = self._input[self._pos:] + self._pos += len(input) + + self._token, remaining = self._herelexer.add(input, eof) + + #Reset here-document state + self._herelexer = None + heredoc, self._heredoc = self._heredoc, HereDoc(None) + if remaining: + self._input[self._pos:self._pos] = list(remaining) + self._state = self.ST_NORMAL + + #Push pending tokens + heredoc.pendings[:0] = [(self._token, self._type, heredoc.name)] + for token, type, delim in heredoc.pendings: + self._token = token + self._type = type + self._push_token(delim) + + def _push_token(self, delim): + if not self._token: + return 0 + + if self._heredoc.op is not None: + if self._heredoc.name is None: + #Here-document name + if self._type!=TK_TOKEN: + raise ShellSyntaxError("expecting here-document name, got '%s'" % self._token) + self._heredoc.name = unquote_wordtree(make_wordtree(self._token)) + self._type = TK_HERENAME + else: + #Capture all tokens until the newline starting the here-document + if self._type==TK_NEWLINE: + assert self._state==self.ST_NORMAL + self._state = self.ST_HEREDOC + + self._heredoc.pendings.append((self._token, self._type, delim)) + self._token = '' + self._type = TK_TOKEN + return 1 + + # BEWARE: do not change parser state from here to the end of the function: + # when parsing between an here-document operator to the end of the line + # tokens are stored in self._heredoc.pendings. Therefore, they will not + # reach the section below. + + #Check operators + if self._type==TK_OP: + #False positive because of partial op matching + op = is_op(self._token) + if not op: + self._type = TK_TOKEN + else: + #Map to the specific operator + self._type = op + if self._token in ('<<', '<<-'): + #Done here rather than in _parse_op because there is no need + #to change the parser state since we are still waiting for + #the here-document name + if self._heredoc.op is not None: + raise ShellSyntaxError("syntax error near token '%s'" % self._token) + assert self._heredoc.op is None + self._heredoc.op = self._token + + if self._type==TK_TOKEN: + if '=' in self._token and not delim: + if self._token.startswith('='): + #Token is a WORD... a TOKEN that is. + pass + else: + prev = self._token[:self._token.find('=')] + if is_name(prev): + self._type = TK_ASSIGNMENT + else: + #Just a token (unspecified) + pass + else: + reserved = get_reserved(self._token) + if reserved is not None: + if reserved=='In' and self._for_count!=2: + #Sorry, not a reserved word after all + pass + else: + self._type = reserved + if reserved in ('For', 'Case'): + self._for_count = 0 + elif are_digits(self._token) and delim in ('<', '>'): + #Detect IO_NUMBER + self._type = TK_IONUMBER + elif self._token==';': + self._type = TK_COMMA + elif self._token=='&': + self._type = TK_AMPERSAND + elif self._type==TK_COMMENT: + #Comments are not part of sh grammar, ignore them + self._token = '' + self._type = TK_TOKEN + return 0 + + if self._for_count is not None: + #Track token count in 'For' expression to detect 'In' reserved words. + #Can only be in third position, no need to go beyond + self._for_count += 1 + if self._for_count==3: + self._for_count = None + + self.on_token((self._token, self._type)) + self._token = '' + self._type = TK_TOKEN + return 1 + + def on_token(self, token): + raise NotImplementedError + + +tokens = [ + TK_TOKEN, +# To silence yacc unused token warnings +# TK_COMMENT, + TK_NEWLINE, + TK_IONUMBER, + TK_ASSIGNMENT, + TK_HERENAME, +] + +#Add specific operators +tokens += _OPERATORS.values() +#Add reserved words +tokens += _RESERVEDS.values() + +class PLYLexer(Lexer): + """Bridge Lexer and PLY lexer interface.""" + def __init__(self): + Lexer.__init__(self) + self._tokens = [] + self._current = 0 + self.lineno = 0 + + def on_token(self, token): + value, type = token + + self.lineno = 0 + t = lex.LexToken() + t.value = value + t.type = type + t.lexer = self + t.lexpos = 0 + t.lineno = 0 + + self._tokens.append(t) + + def is_empty(self): + return not bool(self._tokens) + + #PLY compliant interface + def token(self): + if self._current>=len(self._tokens): + return None + t = self._tokens[self._current] + self._current += 1 + return t + + +def get_tokens(s): + """Parse the input string and return a tuple (tokens, unprocessed) where + tokens is a list of parsed tokens and unprocessed is the part of the input + string left untouched by the lexer. + """ + lexer = PLYLexer() + untouched = lexer.add(s, True) + tokens = [] + while 1: + token = lexer.token() + if token is None: + break + tokens.append(token) + + tokens = [(t.value, t.type) for t in tokens] + return tokens, untouched diff --git a/import-layers/yocto-poky/bitbake/lib/bb/pysh/pyshyacc.py b/import-layers/yocto-poky/bitbake/lib/bb/pysh/pyshyacc.py new file mode 100644 index 000000000..e8e80aac4 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/pysh/pyshyacc.py @@ -0,0 +1,779 @@ +# pyshyacc.py - PLY grammar definition for pysh +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +"""PLY grammar file. +""" +import os.path +import sys + +import pyshlex +tokens = pyshlex.tokens + +from ply import yacc +import sherrors + +class IORedirect: + def __init__(self, op, filename, io_number=None): + self.op = op + self.filename = filename + self.io_number = io_number + +class HereDocument: + def __init__(self, op, name, content, io_number=None): + self.op = op + self.name = name + self.content = content + self.io_number = io_number + +def make_io_redirect(p): + """Make an IORedirect instance from the input 'io_redirect' production.""" + name, io_number, io_target = p + assert name=='io_redirect' + + if io_target[0]=='io_file': + io_type, io_op, io_file = io_target + return IORedirect(io_op, io_file, io_number) + elif io_target[0]=='io_here': + io_type, io_op, io_name, io_content = io_target + return HereDocument(io_op, io_name, io_content, io_number) + else: + assert False, "Invalid IO redirection token %s" % repr(io_type) + +class SimpleCommand: + """ + assigns contains (name, value) pairs. + """ + def __init__(self, words, redirs, assigns): + self.words = list(words) + self.redirs = list(redirs) + self.assigns = list(assigns) + +class Pipeline: + def __init__(self, commands, reverse_status=False): + self.commands = list(commands) + assert self.commands #Grammar forbids this + self.reverse_status = reverse_status + +class AndOr: + def __init__(self, op, left, right): + self.op = str(op) + self.left = left + self.right = right + +class ForLoop: + def __init__(self, name, items, cmds): + self.name = str(name) + self.items = list(items) + self.cmds = list(cmds) + +class WhileLoop: + def __init__(self, condition, cmds): + self.condition = list(condition) + self.cmds = list(cmds) + +class UntilLoop: + def __init__(self, condition, cmds): + self.condition = list(condition) + self.cmds = list(cmds) + +class FunDef: + def __init__(self, name, body): + self.name = str(name) + self.body = body + +class BraceGroup: + def __init__(self, cmds): + self.cmds = list(cmds) + +class IfCond: + def __init__(self, cond, if_cmds, else_cmds): + self.cond = list(cond) + self.if_cmds = if_cmds + self.else_cmds = else_cmds + +class Case: + def __init__(self, name, items): + self.name = name + self.items = items + +class SubShell: + def __init__(self, cmds): + self.cmds = cmds + +class RedirectList: + def __init__(self, cmd, redirs): + self.cmd = cmd + self.redirs = list(redirs) + +def get_production(productions, ptype): + """productions must be a list of production tuples like (name, obj) where + name is the production string identifier. + Return the first production named 'ptype'. Raise KeyError if None can be + found. + """ + for production in productions: + if production is not None and production[0]==ptype: + return production + raise KeyError(ptype) + +#------------------------------------------------------------------------------- +# PLY grammar definition +#------------------------------------------------------------------------------- + +def p_multiple_commands(p): + """multiple_commands : newline_sequence + | complete_command + | multiple_commands complete_command""" + if len(p)==2: + if p[1] is not None: + p[0] = [p[1]] + else: + p[0] = [] + else: + p[0] = p[1] + [p[2]] + +def p_complete_command(p): + """complete_command : list separator + | list""" + if len(p)==3 and p[2] and p[2][1] == '&': + p[0] = ('async', p[1]) + else: + p[0] = p[1] + +def p_list(p): + """list : list separator_op and_or + | and_or""" + if len(p)==2: + p[0] = [p[1]] + else: + #if p[2]!=';': + # raise NotImplementedError('AND-OR list asynchronous execution is not implemented') + p[0] = p[1] + [p[3]] + +def p_and_or(p): + """and_or : pipeline + | and_or AND_IF linebreak pipeline + | and_or OR_IF linebreak pipeline""" + if len(p)==2: + p[0] = p[1] + else: + p[0] = ('and_or', AndOr(p[2], p[1], p[4])) + +def p_maybe_bang_word(p): + """maybe_bang_word : Bang""" + p[0] = ('maybe_bang_word', p[1]) + +def p_pipeline(p): + """pipeline : pipe_sequence + | bang_word pipe_sequence""" + if len(p)==3: + p[0] = ('pipeline', Pipeline(p[2][1:], True)) + else: + p[0] = ('pipeline', Pipeline(p[1][1:])) + +def p_pipe_sequence(p): + """pipe_sequence : command + | pipe_sequence PIPE linebreak command""" + if len(p)==2: + p[0] = ['pipe_sequence', p[1]] + else: + p[0] = p[1] + [p[4]] + +def p_command(p): + """command : simple_command + | compound_command + | compound_command redirect_list + | function_definition""" + + if p[1][0] in ( 'simple_command', + 'for_clause', + 'while_clause', + 'until_clause', + 'case_clause', + 'if_clause', + 'function_definition', + 'subshell', + 'brace_group',): + if len(p) == 2: + p[0] = p[1] + else: + p[0] = ('redirect_list', RedirectList(p[1], p[2][1:])) + else: + raise NotImplementedError('%s command is not implemented' % repr(p[1][0])) + +def p_compound_command(p): + """compound_command : brace_group + | subshell + | for_clause + | case_clause + | if_clause + | while_clause + | until_clause""" + p[0] = p[1] + +def p_subshell(p): + """subshell : LPARENS compound_list RPARENS""" + p[0] = ('subshell', SubShell(p[2][1:])) + +def p_compound_list(p): + """compound_list : term + | newline_list term + | term separator + | newline_list term separator""" + productions = p[1:] + try: + sep = get_production(productions, 'separator') + if sep[1]!=';': + raise NotImplementedError() + except KeyError: + pass + term = get_production(productions, 'term') + p[0] = ['compound_list'] + term[1:] + +def p_term(p): + """term : term separator and_or + | and_or""" + if len(p)==2: + p[0] = ['term', p[1]] + else: + if p[2] is not None and p[2][1] == '&': + p[0] = ['term', ('async', p[1][1:])] + [p[3]] + else: + p[0] = p[1] + [p[3]] + +def p_maybe_for_word(p): + # Rearrange 'For' priority wrt TOKEN. See p_for_word + """maybe_for_word : For""" + p[0] = ('maybe_for_word', p[1]) + +def p_for_clause(p): + """for_clause : for_word name linebreak do_group + | for_word name linebreak in sequential_sep do_group + | for_word name linebreak in wordlist sequential_sep do_group""" + productions = p[1:] + do_group = get_production(productions, 'do_group') + try: + items = get_production(productions, 'in')[1:] + except KeyError: + raise NotImplementedError('"in" omission is not implemented') + + try: + items = get_production(productions, 'wordlist')[1:] + except KeyError: + items = [] + + name = p[2] + p[0] = ('for_clause', ForLoop(name, items, do_group[1:])) + +def p_name(p): + """name : token""" #Was NAME instead of token + p[0] = p[1] + +def p_in(p): + """in : In""" + p[0] = ('in', p[1]) + +def p_wordlist(p): + """wordlist : wordlist token + | token""" + if len(p)==2: + p[0] = ['wordlist', ('TOKEN', p[1])] + else: + p[0] = p[1] + [('TOKEN', p[2])] + +def p_case_clause(p): + """case_clause : Case token linebreak in linebreak case_list Esac + | Case token linebreak in linebreak case_list_ns Esac + | Case token linebreak in linebreak Esac""" + if len(p) < 8: + items = [] + else: + items = p[6][1:] + name = p[2] + p[0] = ('case_clause', Case(name, [c[1] for c in items])) + +def p_case_list_ns(p): + """case_list_ns : case_list case_item_ns + | case_item_ns""" + p_case_list(p) + +def p_case_list(p): + """case_list : case_list case_item + | case_item""" + if len(p)==2: + p[0] = ['case_list', p[1]] + else: + p[0] = p[1] + [p[2]] + +def p_case_item_ns(p): + """case_item_ns : pattern RPARENS linebreak + | pattern RPARENS compound_list linebreak + | LPARENS pattern RPARENS linebreak + | LPARENS pattern RPARENS compound_list linebreak""" + p_case_item(p) + +def p_case_item(p): + """case_item : pattern RPARENS linebreak DSEMI linebreak + | pattern RPARENS compound_list DSEMI linebreak + | LPARENS pattern RPARENS linebreak DSEMI linebreak + | LPARENS pattern RPARENS compound_list DSEMI linebreak""" + if len(p) < 7: + name = p[1][1:] + else: + name = p[2][1:] + + try: + cmds = get_production(p[1:], "compound_list")[1:] + except KeyError: + cmds = [] + + p[0] = ('case_item', (name, cmds)) + +def p_pattern(p): + """pattern : token + | pattern PIPE token""" + if len(p)==2: + p[0] = ['pattern', ('TOKEN', p[1])] + else: + p[0] = p[1] + [('TOKEN', p[2])] + +def p_maybe_if_word(p): + # Rearrange 'If' priority wrt TOKEN. See p_if_word + """maybe_if_word : If""" + p[0] = ('maybe_if_word', p[1]) + +def p_maybe_then_word(p): + # Rearrange 'Then' priority wrt TOKEN. See p_then_word + """maybe_then_word : Then""" + p[0] = ('maybe_then_word', p[1]) + +def p_if_clause(p): + """if_clause : if_word compound_list then_word compound_list else_part Fi + | if_word compound_list then_word compound_list Fi""" + else_part = [] + if len(p)==7: + else_part = p[5] + p[0] = ('if_clause', IfCond(p[2][1:], p[4][1:], else_part)) + +def p_else_part(p): + """else_part : Elif compound_list then_word compound_list else_part + | Elif compound_list then_word compound_list + | Else compound_list""" + if len(p)==3: + p[0] = p[2][1:] + else: + else_part = [] + if len(p)==6: + else_part = p[5] + p[0] = ('elif', IfCond(p[2][1:], p[4][1:], else_part)) + +def p_while_clause(p): + """while_clause : While compound_list do_group""" + p[0] = ('while_clause', WhileLoop(p[2][1:], p[3][1:])) + +def p_maybe_until_word(p): + # Rearrange 'Until' priority wrt TOKEN. See p_until_word + """maybe_until_word : Until""" + p[0] = ('maybe_until_word', p[1]) + +def p_until_clause(p): + """until_clause : until_word compound_list do_group""" + p[0] = ('until_clause', UntilLoop(p[2][1:], p[3][1:])) + +def p_function_definition(p): + """function_definition : fname LPARENS RPARENS linebreak function_body""" + p[0] = ('function_definition', FunDef(p[1], p[5])) + +def p_function_body(p): + """function_body : compound_command + | compound_command redirect_list""" + if len(p)!=2: + raise NotImplementedError('functions redirections lists are not implemented') + p[0] = p[1] + +def p_fname(p): + """fname : TOKEN""" #Was NAME instead of token + p[0] = p[1] + +def p_brace_group(p): + """brace_group : Lbrace compound_list Rbrace""" + p[0] = ('brace_group', BraceGroup(p[2][1:])) + +def p_maybe_done_word(p): + #See p_assignment_word for details. + """maybe_done_word : Done""" + p[0] = ('maybe_done_word', p[1]) + +def p_maybe_do_word(p): + """maybe_do_word : Do""" + p[0] = ('maybe_do_word', p[1]) + +def p_do_group(p): + """do_group : do_word compound_list done_word""" + #Do group contains a list of AndOr + p[0] = ['do_group'] + p[2][1:] + +def p_simple_command(p): + """simple_command : cmd_prefix cmd_word cmd_suffix + | cmd_prefix cmd_word + | cmd_prefix + | cmd_name cmd_suffix + | cmd_name""" + words, redirs, assigns = [], [], [] + for e in p[1:]: + name = e[0] + if name in ('cmd_prefix', 'cmd_suffix'): + for sube in e[1:]: + subname = sube[0] + if subname=='io_redirect': + redirs.append(make_io_redirect(sube)) + elif subname=='ASSIGNMENT_WORD': + assigns.append(sube) + else: + words.append(sube) + elif name in ('cmd_word', 'cmd_name'): + words.append(e) + + cmd = SimpleCommand(words, redirs, assigns) + p[0] = ('simple_command', cmd) + +def p_cmd_name(p): + """cmd_name : TOKEN""" + p[0] = ('cmd_name', p[1]) + +def p_cmd_word(p): + """cmd_word : token""" + p[0] = ('cmd_word', p[1]) + +def p_maybe_assignment_word(p): + #See p_assignment_word for details. + """maybe_assignment_word : ASSIGNMENT_WORD""" + p[0] = ('maybe_assignment_word', p[1]) + +def p_cmd_prefix(p): + """cmd_prefix : io_redirect + | cmd_prefix io_redirect + | assignment_word + | cmd_prefix assignment_word""" + try: + prefix = get_production(p[1:], 'cmd_prefix') + except KeyError: + prefix = ['cmd_prefix'] + + try: + value = get_production(p[1:], 'assignment_word')[1] + value = ('ASSIGNMENT_WORD', value.split('=', 1)) + except KeyError: + value = get_production(p[1:], 'io_redirect') + p[0] = prefix + [value] + +def p_cmd_suffix(p): + """cmd_suffix : io_redirect + | cmd_suffix io_redirect + | token + | cmd_suffix token + | maybe_for_word + | cmd_suffix maybe_for_word + | maybe_done_word + | cmd_suffix maybe_done_word + | maybe_do_word + | cmd_suffix maybe_do_word + | maybe_until_word + | cmd_suffix maybe_until_word + | maybe_assignment_word + | cmd_suffix maybe_assignment_word + | maybe_if_word + | cmd_suffix maybe_if_word + | maybe_then_word + | cmd_suffix maybe_then_word + | maybe_bang_word + | cmd_suffix maybe_bang_word""" + try: + suffix = get_production(p[1:], 'cmd_suffix') + token = p[2] + except KeyError: + suffix = ['cmd_suffix'] + token = p[1] + + if isinstance(token, tuple): + if token[0]=='io_redirect': + p[0] = suffix + [token] + else: + #Convert maybe_* to TOKEN if necessary + p[0] = suffix + [('TOKEN', token[1])] + else: + p[0] = suffix + [('TOKEN', token)] + +def p_redirect_list(p): + """redirect_list : io_redirect + | redirect_list io_redirect""" + if len(p) == 2: + p[0] = ['redirect_list', make_io_redirect(p[1])] + else: + p[0] = p[1] + [make_io_redirect(p[2])] + +def p_io_redirect(p): + """io_redirect : io_file + | IO_NUMBER io_file + | io_here + | IO_NUMBER io_here""" + if len(p)==3: + p[0] = ('io_redirect', p[1], p[2]) + else: + p[0] = ('io_redirect', None, p[1]) + +def p_io_file(p): + #Return the tuple (operator, filename) + """io_file : LESS filename + | LESSAND filename + | GREATER filename + | GREATAND filename + | DGREAT filename + | LESSGREAT filename + | CLOBBER filename""" + #Extract the filename from the file + p[0] = ('io_file', p[1], p[2][1]) + +def p_filename(p): + #Return the filename + """filename : TOKEN""" + p[0] = ('filename', p[1]) + +def p_io_here(p): + """io_here : DLESS here_end + | DLESSDASH here_end""" + p[0] = ('io_here', p[1], p[2][1], p[2][2]) + +def p_here_end(p): + """here_end : HERENAME TOKEN""" + p[0] = ('here_document', p[1], p[2]) + +def p_newline_sequence(p): + # Nothing in the grammar can handle leading NEWLINE productions, so add + # this one with the lowest possible priority relatively to newline_list. + """newline_sequence : newline_list""" + p[0] = None + +def p_newline_list(p): + """newline_list : NEWLINE + | newline_list NEWLINE""" + p[0] = None + +def p_linebreak(p): + """linebreak : newline_list + | empty""" + p[0] = None + +def p_separator_op(p): + """separator_op : COMMA + | AMP""" + p[0] = p[1] + +def p_separator(p): + """separator : separator_op linebreak + | newline_list""" + if len(p)==2: + #Ignore newlines + p[0] = None + else: + #Keep the separator operator + p[0] = ('separator', p[1]) + +def p_sequential_sep(p): + """sequential_sep : COMMA linebreak + | newline_list""" + p[0] = None + +# Low priority TOKEN => for_word conversion. +# Let maybe_for_word be used as a token when necessary in higher priority +# rules. +def p_for_word(p): + """for_word : maybe_for_word""" + p[0] = p[1] + +def p_if_word(p): + """if_word : maybe_if_word""" + p[0] = p[1] + +def p_then_word(p): + """then_word : maybe_then_word""" + p[0] = p[1] + +def p_done_word(p): + """done_word : maybe_done_word""" + p[0] = p[1] + +def p_do_word(p): + """do_word : maybe_do_word""" + p[0] = p[1] + +def p_until_word(p): + """until_word : maybe_until_word""" + p[0] = p[1] + +def p_assignment_word(p): + """assignment_word : maybe_assignment_word""" + p[0] = ('assignment_word', p[1][1]) + +def p_bang_word(p): + """bang_word : maybe_bang_word""" + p[0] = ('bang_word', p[1][1]) + +def p_token(p): + """token : TOKEN + | Fi""" + p[0] = p[1] + +def p_empty(p): + 'empty :' + p[0] = None + +# Error rule for syntax errors +def p_error(p): + msg = [] + w = msg.append + w('%r\n' % p) + w('followed by:\n') + for i in range(5): + n = yacc.token() + if not n: + break + w(' %r\n' % n) + raise sherrors.ShellSyntaxError(''.join(msg)) + +# Build the parser +try: + import pyshtables +except ImportError: + outputdir = os.path.dirname(__file__) + if not os.access(outputdir, os.W_OK): + outputdir = '' + yacc.yacc(tabmodule = 'pyshtables', outputdir = outputdir, debug = 0) +else: + yacc.yacc(tabmodule = 'pysh.pyshtables', write_tables = 0, debug = 0) + + +def parse(input, eof=False, debug=False): + """Parse a whole script at once and return the generated AST and unconsumed + data in a tuple. + + NOTE: eof is probably meaningless for now, the parser being unable to work + in pull mode. It should be set to True. + """ + lexer = pyshlex.PLYLexer() + remaining = lexer.add(input, eof) + if lexer.is_empty(): + return [], remaining + if debug: + debug = 2 + return yacc.parse(lexer=lexer, debug=debug), remaining + +#------------------------------------------------------------------------------- +# AST rendering helpers +#------------------------------------------------------------------------------- + +def format_commands(v): + """Return a tree made of strings and lists. Make command trees easier to + display. + """ + if isinstance(v, list): + return [format_commands(c) for c in v] + if isinstance(v, tuple): + if len(v)==2 and isinstance(v[0], str) and not isinstance(v[1], str): + if v[0] == 'async': + return ['AsyncList', map(format_commands, v[1])] + else: + #Avoid decomposing tuples like ('pipeline', Pipeline(...)) + return format_commands(v[1]) + return format_commands(list(v)) + elif isinstance(v, IfCond): + name = ['IfCond'] + name += ['if', map(format_commands, v.cond)] + name += ['then', map(format_commands, v.if_cmds)] + name += ['else', map(format_commands, v.else_cmds)] + return name + elif isinstance(v, ForLoop): + name = ['ForLoop'] + name += [repr(v.name)+' in ', map(str, v.items)] + name += ['commands', map(format_commands, v.cmds)] + return name + elif isinstance(v, AndOr): + return [v.op, format_commands(v.left), format_commands(v.right)] + elif isinstance(v, Pipeline): + name = 'Pipeline' + if v.reverse_status: + name = '!' + name + return [name, format_commands(v.commands)] + elif isinstance(v, Case): + name = ['Case'] + name += [v.name, format_commands(v.items)] + elif isinstance(v, SimpleCommand): + name = ['SimpleCommand'] + if v.words: + name += ['words', map(str, v.words)] + if v.assigns: + assigns = [tuple(a[1]) for a in v.assigns] + name += ['assigns', map(str, assigns)] + if v.redirs: + name += ['redirs', map(format_commands, v.redirs)] + return name + elif isinstance(v, RedirectList): + name = ['RedirectList'] + if v.redirs: + name += ['redirs', map(format_commands, v.redirs)] + name += ['command', format_commands(v.cmd)] + return name + elif isinstance(v, IORedirect): + return ' '.join(map(str, (v.io_number, v.op, v.filename))) + elif isinstance(v, HereDocument): + return ' '.join(map(str, (v.io_number, v.op, repr(v.name), repr(v.content)))) + elif isinstance(v, SubShell): + return ['SubShell', map(format_commands, v.cmds)] + else: + return repr(v) + +def print_commands(cmds, output=sys.stdout): + """Pretty print a command tree.""" + def print_tree(cmd, spaces, output): + if isinstance(cmd, list): + for c in cmd: + print_tree(c, spaces + 3, output) + else: + print >>output, ' '*spaces + str(cmd) + + formatted = format_commands(cmds) + print_tree(formatted, 0, output) + + +def stringify_commands(cmds): + """Serialize a command tree as a string. + + Returned string is not pretty and is currently used for unit tests only. + """ + def stringify(value): + output = [] + if isinstance(value, list): + formatted = [] + for v in value: + formatted.append(stringify(v)) + formatted = ' '.join(formatted) + output.append(''.join(['<', formatted, '>'])) + else: + output.append(value) + return ' '.join(output) + + return stringify(format_commands(cmds)) + + +def visit_commands(cmds, callable): + """Visit the command tree and execute callable on every Pipeline and + SimpleCommand instances. + """ + if isinstance(cmds, (tuple, list)): + map(lambda c: visit_commands(c,callable), cmds) + elif isinstance(cmds, (Pipeline, SimpleCommand)): + callable(cmds) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/pysh/sherrors.py b/import-layers/yocto-poky/bitbake/lib/bb/pysh/sherrors.py new file mode 100644 index 000000000..49d0533de --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/pysh/sherrors.py @@ -0,0 +1,41 @@ +# sherrors.py - shell errors and signals +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +"""Define shell exceptions and error codes. +""" + +class ShellError(Exception): + pass + +class ShellSyntaxError(ShellError): + pass + +class UtilityError(ShellError): + """Raised upon utility syntax error (option or operand error).""" + pass + +class ExpansionError(ShellError): + pass + +class CommandNotFound(ShellError): + """Specified command was not found.""" + pass + +class RedirectionError(ShellError): + pass + +class VarAssignmentError(ShellError): + """Variable assignment error.""" + pass + +class ExitSignal(ShellError): + """Exit signal.""" + pass + +class ReturnSignal(ShellError): + """Exit signal.""" + pass diff --git a/import-layers/yocto-poky/bitbake/lib/bb/pysh/subprocess_fix.py b/import-layers/yocto-poky/bitbake/lib/bb/pysh/subprocess_fix.py new file mode 100644 index 000000000..46eca2280 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/pysh/subprocess_fix.py @@ -0,0 +1,77 @@ +# subprocess - Subprocesses with accessible I/O streams +# +# For more information about this module, see PEP 324. +# +# This module should remain compatible with Python 2.2, see PEP 291. +# +# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> +# +# Licensed to PSF under a Contributor Agreement. +# See http://www.python.org/2.4/license for licensing details. + +def list2cmdline(seq): + """ + Translate a sequence of arguments into a command line + string, using the same rules as the MS C runtime: + + 1) Arguments are delimited by white space, which is either a + space or a tab. + + 2) A string surrounded by double quotation marks is + interpreted as a single argument, regardless of white space + contained within. A quoted string can be embedded in an + argument. + + 3) A double quotation mark preceded by a backslash is + interpreted as a literal double quotation mark. + + 4) Backslashes are interpreted literally, unless they + immediately precede a double quotation mark. + + 5) If backslashes immediately precede a double quotation mark, + every pair of backslashes is interpreted as a literal + backslash. If the number of backslashes is odd, the last + backslash escapes the next double quotation mark as + described in rule 3. + """ + + # See + # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp + result = [] + needquote = False + for arg in seq: + bs_buf = [] + + # Add a space to separate this argument from the others + if result: + result.append(' ') + + needquote = (" " in arg) or ("\t" in arg) or ("|" in arg) or arg == "" + if needquote: + result.append('"') + + for c in arg: + if c == '\\': + # Don't know if we need to double yet. + bs_buf.append(c) + elif c == '"': + # Double backspaces. + result.append('\\' * len(bs_buf)*2) + bs_buf = [] + result.append('\\"') + else: + # Normal char + if bs_buf: + result.extend(bs_buf) + bs_buf = [] + result.append(c) + + # Add remaining backspaces, if any. + if bs_buf: + result.extend(bs_buf) + + if needquote: + result.extend(bs_buf) + result.append('"') + + return ''.join(result) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py b/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py new file mode 100644 index 000000000..e1b9b2e66 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py @@ -0,0 +1,2285 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'RunQueue' implementation + +Handles preparation and execution of a queue of tasks +""" + +# Copyright (C) 2006-2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import copy +import os +import sys +import signal +import stat +import fcntl +import errno +import logging +import re +import bb +from bb import msg, data, event +from bb import monitordisk +import subprocess + +try: + import cPickle as pickle +except ImportError: + import pickle + +bblogger = logging.getLogger("BitBake") +logger = logging.getLogger("BitBake.RunQueue") + +__find_md5__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])' ) + +class RunQueueStats: + """ + Holds statistics on the tasks handled by the associated runQueue + """ + def __init__(self, total): + self.completed = 0 + self.skipped = 0 + self.failed = 0 + self.active = 0 + self.total = total + + def copy(self): + obj = self.__class__(self.total) + obj.__dict__.update(self.__dict__) + return obj + + def taskFailed(self): + self.active = self.active - 1 + self.failed = self.failed + 1 + + def taskCompleted(self, number = 1): + self.active = self.active - number + self.completed = self.completed + number + + def taskSkipped(self, number = 1): + self.active = self.active + number + self.skipped = self.skipped + number + + def taskActive(self): + self.active = self.active + 1 + +# These values indicate the next step due to be run in the +# runQueue state machine +runQueuePrepare = 2 +runQueueSceneInit = 3 +runQueueSceneRun = 4 +runQueueRunInit = 5 +runQueueRunning = 6 +runQueueFailed = 7 +runQueueCleanUp = 8 +runQueueComplete = 9 + +class RunQueueScheduler(object): + """ + Control the order tasks are scheduled in. + """ + name = "basic" + + def __init__(self, runqueue, rqdata): + """ + The default scheduler just returns the first buildable task (the + priority map is sorted by task number) + """ + self.rq = runqueue + self.rqdata = rqdata + self.numTasks = len(self.rqdata.runq_fnid) + + self.prio_map = [] + self.prio_map.extend(range(self.numTasks)) + + self.buildable = [] + self.stamps = {} + for taskid in xrange(self.numTasks): + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]] + taskname = self.rqdata.runq_task[taskid] + self.stamps[taskid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) + if self.rq.runq_buildable[taskid] == 1: + self.buildable.append(taskid) + + self.rev_prio_map = None + + def next_buildable_task(self): + """ + Return the id of the first task we find that is buildable + """ + self.buildable = [x for x in self.buildable if not self.rq.runq_running[x] == 1] + if not self.buildable: + return None + if len(self.buildable) == 1: + taskid = self.buildable[0] + stamp = self.stamps[taskid] + if stamp not in self.rq.build_stamps.itervalues(): + return taskid + + if not self.rev_prio_map: + self.rev_prio_map = range(self.numTasks) + for taskid in xrange(self.numTasks): + self.rev_prio_map[self.prio_map[taskid]] = taskid + + best = None + bestprio = None + for taskid in self.buildable: + prio = self.rev_prio_map[taskid] + if bestprio is None or bestprio > prio: + stamp = self.stamps[taskid] + if stamp in self.rq.build_stamps.itervalues(): + continue + bestprio = prio + best = taskid + + return best + + def next(self): + """ + Return the id of the task we should build next + """ + if self.rq.stats.active < self.rq.number_tasks: + return self.next_buildable_task() + + def newbuilable(self, task): + self.buildable.append(task) + +class RunQueueSchedulerSpeed(RunQueueScheduler): + """ + A scheduler optimised for speed. The priority map is sorted by task weight, + heavier weighted tasks (tasks needed by the most other tasks) are run first. + """ + name = "speed" + + def __init__(self, runqueue, rqdata): + """ + The priority map is sorted by task weight. + """ + RunQueueScheduler.__init__(self, runqueue, rqdata) + + sortweight = sorted(copy.deepcopy(self.rqdata.runq_weight)) + copyweight = copy.deepcopy(self.rqdata.runq_weight) + self.prio_map = [] + + for weight in sortweight: + idx = copyweight.index(weight) + self.prio_map.append(idx) + copyweight[idx] = -1 + + self.prio_map.reverse() + +class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): + """ + A scheduler optimised to complete .bb files are quickly as possible. The + priority map is sorted by task weight, but then reordered so once a given + .bb file starts to build, it's completed as quickly as possible. This works + well where disk space is at a premium and classes like OE's rm_work are in + force. + """ + name = "completion" + + def __init__(self, runqueue, rqdata): + RunQueueSchedulerSpeed.__init__(self, runqueue, rqdata) + + #FIXME - whilst this groups all fnids together it does not reorder the + #fnid groups optimally. + + basemap = copy.deepcopy(self.prio_map) + self.prio_map = [] + while (len(basemap) > 0): + entry = basemap.pop(0) + self.prio_map.append(entry) + fnid = self.rqdata.runq_fnid[entry] + todel = [] + for entry in basemap: + entry_fnid = self.rqdata.runq_fnid[entry] + if entry_fnid == fnid: + todel.append(basemap.index(entry)) + self.prio_map.append(entry) + todel.reverse() + for idx in todel: + del basemap[idx] + +class RunQueueData: + """ + BitBake Run Queue implementation + """ + def __init__(self, rq, cooker, cfgData, dataCache, taskData, targets): + self.cooker = cooker + self.dataCache = dataCache + self.taskData = taskData + self.targets = targets + self.rq = rq + self.warn_multi_bb = False + + self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or "" + self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split() + + self.reset() + + def reset(self): + self.runq_fnid = [] + self.runq_task = [] + self.runq_depends = [] + self.runq_revdeps = [] + self.runq_hash = [] + + def runq_depends_names(self, ids): + import re + ret = [] + for id in self.runq_depends[ids]: + nam = os.path.basename(self.get_user_idstring(id)) + nam = re.sub("_[^,]*,", ",", nam) + ret.extend([nam]) + return ret + + def get_task_name(self, task): + return self.runq_task[task] + + def get_task_file(self, task): + return self.taskData.fn_index[self.runq_fnid[task]] + + def get_task_hash(self, task): + return self.runq_hash[task] + + def get_user_idstring(self, task, task_name_suffix = ""): + fn = self.taskData.fn_index[self.runq_fnid[task]] + taskname = self.runq_task[task] + task_name_suffix + return "%s, %s" % (fn, taskname) + + def get_short_user_idstring(self, task, task_name_suffix = ""): + fn = self.taskData.fn_index[self.runq_fnid[task]] + pn = self.dataCache.pkg_fn[fn] + taskname = self.runq_task[task] + task_name_suffix + return "%s:%s" % (pn, taskname) + + + def get_task_id(self, fnid, taskname): + for listid in xrange(len(self.runq_fnid)): + if self.runq_fnid[listid] == fnid and self.runq_task[listid] == taskname: + return listid + return None + + def circular_depchains_handler(self, tasks): + """ + Some tasks aren't buildable, likely due to circular dependency issues. + Identify the circular dependencies and print them in a user readable format. + """ + from copy import deepcopy + + valid_chains = [] + explored_deps = {} + msgs = [] + + def chain_reorder(chain): + """ + Reorder a dependency chain so the lowest task id is first + """ + lowest = 0 + new_chain = [] + for entry in xrange(len(chain)): + if chain[entry] < chain[lowest]: + lowest = entry + new_chain.extend(chain[lowest:]) + new_chain.extend(chain[:lowest]) + return new_chain + + def chain_compare_equal(chain1, chain2): + """ + Compare two dependency chains and see if they're the same + """ + if len(chain1) != len(chain2): + return False + for index in xrange(len(chain1)): + if chain1[index] != chain2[index]: + return False + return True + + def chain_array_contains(chain, chain_array): + """ + Return True if chain_array contains chain + """ + for ch in chain_array: + if chain_compare_equal(ch, chain): + return True + return False + + def find_chains(taskid, prev_chain): + prev_chain.append(taskid) + total_deps = [] + total_deps.extend(self.runq_revdeps[taskid]) + for revdep in self.runq_revdeps[taskid]: + if revdep in prev_chain: + idx = prev_chain.index(revdep) + # To prevent duplicates, reorder the chain to start with the lowest taskid + # and search through an array of those we've already printed + chain = prev_chain[idx:] + new_chain = chain_reorder(chain) + if not chain_array_contains(new_chain, valid_chains): + valid_chains.append(new_chain) + msgs.append("Dependency loop #%d found:\n" % len(valid_chains)) + for dep in new_chain: + msgs.append(" Task %s (%s) (dependent Tasks %s)\n" % (dep, self.get_user_idstring(dep), self.runq_depends_names(dep))) + msgs.append("\n") + if len(valid_chains) > 10: + msgs.append("Aborted dependency loops search after 10 matches.\n") + return msgs + continue + scan = False + if revdep not in explored_deps: + scan = True + elif revdep in explored_deps[revdep]: + scan = True + else: + for dep in prev_chain: + if dep in explored_deps[revdep]: + scan = True + if scan: + find_chains(revdep, copy.deepcopy(prev_chain)) + for dep in explored_deps[revdep]: + if dep not in total_deps: + total_deps.append(dep) + + explored_deps[taskid] = total_deps + + for task in tasks: + find_chains(task, []) + + return msgs + + def calculate_task_weights(self, endpoints): + """ + Calculate a number representing the "weight" of each task. Heavier weighted tasks + have more dependencies and hence should be executed sooner for maximum speed. + + This function also sanity checks the task list finding tasks that are not + possible to execute due to circular dependencies. + """ + + numTasks = len(self.runq_fnid) + weight = [] + deps_left = [] + task_done = [] + + for listid in xrange(numTasks): + task_done.append(False) + weight.append(1) + deps_left.append(len(self.runq_revdeps[listid])) + + for listid in endpoints: + weight[listid] = 10 + task_done[listid] = True + + while True: + next_points = [] + for listid in endpoints: + for revdep in self.runq_depends[listid]: + weight[revdep] = weight[revdep] + weight[listid] + deps_left[revdep] = deps_left[revdep] - 1 + if deps_left[revdep] == 0: + next_points.append(revdep) + task_done[revdep] = True + endpoints = next_points + if len(next_points) == 0: + break + + # Circular dependency sanity check + problem_tasks = [] + for task in xrange(numTasks): + if task_done[task] is False or deps_left[task] != 0: + problem_tasks.append(task) + logger.debug(2, "Task %s (%s) is not buildable", task, self.get_user_idstring(task)) + logger.debug(2, "(Complete marker was %s and the remaining dependency count was %s)\n", task_done[task], deps_left[task]) + + if problem_tasks: + message = "Unbuildable tasks were found.\n" + message = message + "These are usually caused by circular dependencies and any circular dependency chains found will be printed below. Increase the debug level to see a list of unbuildable tasks.\n\n" + message = message + "Identifying dependency loops (this may take a short while)...\n" + logger.error(message) + + msgs = self.circular_depchains_handler(problem_tasks) + + message = "\n" + for msg in msgs: + message = message + msg + bb.msg.fatal("RunQueue", message) + + return weight + + def prepare(self): + """ + Turn a set of taskData into a RunQueue and compute data needed + to optimise the execution order. + """ + + runq_build = [] + recursivetasks = {} + recursiveitasks = {} + recursivetasksselfref = set() + + taskData = self.taskData + + if len(taskData.tasks_name) == 0: + # Nothing to do + return 0 + + logger.info("Preparing RunQueue") + + # Step A - Work out a list of tasks to run + # + # Taskdata gives us a list of possible providers for every build and run + # target ordered by priority. It also gives information on each of those + # providers. + # + # To create the actual list of tasks to execute we fix the list of + # providers and then resolve the dependencies into task IDs. This + # process is repeated for each type of dependency (tdepends, deptask, + # rdeptast, recrdeptask, idepends). + + def add_build_dependencies(depids, tasknames, depends): + for depid in depids: + # Won't be in build_targets if ASSUME_PROVIDED + if depid not in taskData.build_targets: + continue + depdata = taskData.build_targets[depid][0] + if depdata is None: + continue + for taskname in tasknames: + taskid = taskData.gettask_id_fromfnid(depdata, taskname) + if taskid is not None: + depends.add(taskid) + + def add_runtime_dependencies(depids, tasknames, depends): + for depid in depids: + if depid not in taskData.run_targets: + continue + depdata = taskData.run_targets[depid][0] + if depdata is None: + continue + for taskname in tasknames: + taskid = taskData.gettask_id_fromfnid(depdata, taskname) + if taskid is not None: + depends.add(taskid) + + def add_resolved_dependencies(depids, tasknames, depends): + for depid in depids: + for taskname in tasknames: + taskid = taskData.gettask_id_fromfnid(depid, taskname) + if taskid is not None: + depends.add(taskid) + + for task in xrange(len(taskData.tasks_name)): + depends = set() + fnid = taskData.tasks_fnid[task] + fn = taskData.fn_index[fnid] + task_deps = self.dataCache.task_deps[fn] + + #logger.debug(2, "Processing %s:%s", fn, taskData.tasks_name[task]) + + if fnid not in taskData.failed_fnids: + + # Resolve task internal dependencies + # + # e.g. addtask before X after Y + depends = set(taskData.tasks_tdepends[task]) + + # Resolve 'deptask' dependencies + # + # e.g. do_sometask[deptask] = "do_someothertask" + # (makes sure sometask runs after someothertask of all DEPENDS) + if 'deptask' in task_deps and taskData.tasks_name[task] in task_deps['deptask']: + tasknames = task_deps['deptask'][taskData.tasks_name[task]].split() + add_build_dependencies(taskData.depids[fnid], tasknames, depends) + + # Resolve 'rdeptask' dependencies + # + # e.g. do_sometask[rdeptask] = "do_someothertask" + # (makes sure sometask runs after someothertask of all RDEPENDS) + if 'rdeptask' in task_deps and taskData.tasks_name[task] in task_deps['rdeptask']: + tasknames = task_deps['rdeptask'][taskData.tasks_name[task]].split() + add_runtime_dependencies(taskData.rdepids[fnid], tasknames, depends) + + # Resolve inter-task dependencies + # + # e.g. do_sometask[depends] = "targetname:do_someothertask" + # (makes sure sometask runs after targetname's someothertask) + idepends = taskData.tasks_idepends[task] + for (depid, idependtask) in idepends: + if depid in taskData.build_targets and not depid in taskData.failed_deps: + # Won't be in build_targets if ASSUME_PROVIDED + depdata = taskData.build_targets[depid][0] + if depdata is not None: + taskid = taskData.gettask_id_fromfnid(depdata, idependtask) + if taskid is None: + bb.msg.fatal("RunQueue", "Task %s in %s depends upon non-existent task %s in %s" % (taskData.tasks_name[task], fn, idependtask, taskData.fn_index[depdata])) + depends.add(taskid) + irdepends = taskData.tasks_irdepends[task] + for (depid, idependtask) in irdepends: + if depid in taskData.run_targets: + # Won't be in run_targets if ASSUME_PROVIDED + depdata = taskData.run_targets[depid][0] + if depdata is not None: + taskid = taskData.gettask_id_fromfnid(depdata, idependtask) + if taskid is None: + bb.msg.fatal("RunQueue", "Task %s in %s rdepends upon non-existent task %s in %s" % (taskData.tasks_name[task], fn, idependtask, taskData.fn_index[depdata])) + depends.add(taskid) + + # Resolve recursive 'recrdeptask' dependencies (Part A) + # + # e.g. do_sometask[recrdeptask] = "do_someothertask" + # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) + # We cover the recursive part of the dependencies below + if 'recrdeptask' in task_deps and taskData.tasks_name[task] in task_deps['recrdeptask']: + tasknames = task_deps['recrdeptask'][taskData.tasks_name[task]].split() + recursivetasks[task] = tasknames + add_build_dependencies(taskData.depids[fnid], tasknames, depends) + add_runtime_dependencies(taskData.rdepids[fnid], tasknames, depends) + if taskData.tasks_name[task] in tasknames: + recursivetasksselfref.add(task) + + if 'recideptask' in task_deps and taskData.tasks_name[task] in task_deps['recideptask']: + recursiveitasks[task] = [] + for t in task_deps['recideptask'][taskData.tasks_name[task]].split(): + newdep = taskData.gettask_id_fromfnid(fnid, t) + recursiveitasks[task].append(newdep) + + self.runq_fnid.append(taskData.tasks_fnid[task]) + self.runq_task.append(taskData.tasks_name[task]) + self.runq_depends.append(depends) + self.runq_revdeps.append(set()) + self.runq_hash.append("") + + runq_build.append(0) + + # Resolve recursive 'recrdeptask' dependencies (Part B) + # + # e.g. do_sometask[recrdeptask] = "do_someothertask" + # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) + # We need to do this separately since we need all of self.runq_depends to be complete before this is processed + extradeps = {} + for task in recursivetasks: + extradeps[task] = set(self.runq_depends[task]) + tasknames = recursivetasks[task] + seendeps = set() + seenfnid = [] + + def generate_recdeps(t): + newdeps = set() + add_resolved_dependencies([taskData.tasks_fnid[t]], tasknames, newdeps) + extradeps[task].update(newdeps) + seendeps.add(t) + newdeps.add(t) + for i in newdeps: + for n in self.runq_depends[i]: + if n not in seendeps: + generate_recdeps(n) + generate_recdeps(task) + + if task in recursiveitasks: + for dep in recursiveitasks[task]: + generate_recdeps(dep) + + # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work + for task in recursivetasks: + extradeps[task].difference_update(recursivetasksselfref) + + for task in xrange(len(taskData.tasks_name)): + # Add in extra dependencies + if task in extradeps: + self.runq_depends[task] = extradeps[task] + # Remove all self references + if task in self.runq_depends[task]: + logger.debug(2, "Task %s (%s %s) contains self reference! %s", task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], self.runq_depends[task]) + self.runq_depends[task].remove(task) + + # Step B - Mark all active tasks + # + # Start with the tasks we were asked to run and mark all dependencies + # as active too. If the task is to be 'forced', clear its stamp. Once + # all active tasks are marked, prune the ones we don't need. + + logger.verbose("Marking Active Tasks") + + def mark_active(listid, depth): + """ + Mark an item as active along with its depends + (calls itself recursively) + """ + + if runq_build[listid] == 1: + return + + runq_build[listid] = 1 + + depends = self.runq_depends[listid] + for depend in depends: + mark_active(depend, depth+1) + + self.target_pairs = [] + for target in self.targets: + targetid = taskData.getbuild_id(target[0]) + + if targetid not in taskData.build_targets: + continue + + if targetid in taskData.failed_deps: + continue + + fnid = taskData.build_targets[targetid][0] + fn = taskData.fn_index[fnid] + task = target[1] + parents = False + if task.endswith('-'): + parents = True + task = task[:-1] + + self.target_pairs.append((fn, task)) + + if fnid in taskData.failed_fnids: + continue + + if task not in taskData.tasks_lookup[fnid]: + import difflib + close_matches = difflib.get_close_matches(task, taskData.tasks_lookup[fnid], cutoff=0.7) + if close_matches: + extra = ". Close matches:\n %s" % "\n ".join(close_matches) + else: + extra = "" + bb.msg.fatal("RunQueue", "Task %s does not exist for target %s%s" % (task, target[0], extra)) + + # For tasks called "XXXX-", ony run their dependencies + listid = taskData.tasks_lookup[fnid][task] + if parents: + for i in self.runq_depends[listid]: + mark_active(i, 1) + else: + mark_active(listid, 1) + + # Step C - Prune all inactive tasks + # + # Once all active tasks are marked, prune the ones we don't need. + + maps = [] + delcount = 0 + for listid in xrange(len(self.runq_fnid)): + if runq_build[listid-delcount] == 1: + maps.append(listid-delcount) + else: + del self.runq_fnid[listid-delcount] + del self.runq_task[listid-delcount] + del self.runq_depends[listid-delcount] + del runq_build[listid-delcount] + del self.runq_revdeps[listid-delcount] + del self.runq_hash[listid-delcount] + delcount = delcount + 1 + maps.append(-1) + + # + # Step D - Sanity checks and computation + # + + # Check to make sure we still have tasks to run + if len(self.runq_fnid) == 0: + if not taskData.abort: + bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.") + else: + bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.") + + logger.verbose("Pruned %s inactive tasks, %s left", delcount, len(self.runq_fnid)) + + # Remap the dependencies to account for the deleted tasks + # Check we didn't delete a task we depend on + for listid in xrange(len(self.runq_fnid)): + newdeps = [] + origdeps = self.runq_depends[listid] + for origdep in origdeps: + if maps[origdep] == -1: + bb.msg.fatal("RunQueue", "Invalid mapping - Should never happen!") + newdeps.append(maps[origdep]) + self.runq_depends[listid] = set(newdeps) + + logger.verbose("Assign Weightings") + + # Generate a list of reverse dependencies to ease future calculations + for listid in xrange(len(self.runq_fnid)): + for dep in self.runq_depends[listid]: + self.runq_revdeps[dep].add(listid) + + # Identify tasks at the end of dependency chains + # Error on circular dependency loops (length two) + endpoints = [] + for listid in xrange(len(self.runq_fnid)): + revdeps = self.runq_revdeps[listid] + if len(revdeps) == 0: + endpoints.append(listid) + for dep in revdeps: + if dep in self.runq_depends[listid]: + #self.dump_data(taskData) + bb.msg.fatal("RunQueue", "Task %s (%s) has circular dependency on %s (%s)" % (taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep], taskData.fn_index[self.runq_fnid[listid]], self.runq_task[listid])) + + logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints)) + + # Calculate task weights + # Check of higher length circular dependencies + self.runq_weight = self.calculate_task_weights(endpoints) + + # Sanity Check - Check for multiple tasks building the same provider + prov_list = {} + seen_fn = [] + for task in xrange(len(self.runq_fnid)): + fn = taskData.fn_index[self.runq_fnid[task]] + if fn in seen_fn: + continue + seen_fn.append(fn) + for prov in self.dataCache.fn_provides[fn]: + if prov not in prov_list: + prov_list[prov] = [fn] + elif fn not in prov_list[prov]: + prov_list[prov].append(fn) + for prov in prov_list: + if len(prov_list[prov]) > 1 and prov not in self.multi_provider_whitelist: + seen_pn = [] + # If two versions of the same PN are being built its fatal, we don't support it. + for fn in prov_list[prov]: + pn = self.dataCache.pkg_fn[fn] + if pn not in seen_pn: + seen_pn.append(pn) + else: + bb.fatal("Multiple versions of %s are due to be built (%s). Only one version of a given PN should be built in any given build. You likely need to set PREFERRED_VERSION_%s to select the correct version or don't depend on multiple versions." % (pn, " ".join(prov_list[prov]), pn)) + msg = "Multiple .bb files are due to be built which each provide %s:\n %s" % (prov, "\n ".join(prov_list[prov])) + # + # Construct a list of things which uniquely depend on each provider + # since this may help the user figure out which dependency is triggering this warning + # + msg += "\nA list of tasks depending on these providers is shown and may help explain where the dependency comes from." + deplist = {} + commondeps = None + for provfn in prov_list[prov]: + deps = set() + for task, fnid in enumerate(self.runq_fnid): + fn = taskData.fn_index[fnid] + if fn != provfn: + continue + for dep in self.runq_revdeps[task]: + fn = taskData.fn_index[self.runq_fnid[dep]] + if fn == provfn: + continue + deps.add(self.get_short_user_idstring(dep)) + if not commondeps: + commondeps = set(deps) + else: + commondeps &= deps + deplist[provfn] = deps + for provfn in deplist: + msg += "\n%s has unique dependees:\n %s" % (provfn, "\n ".join(deplist[provfn] - commondeps)) + # + # Construct a list of provides and runtime providers for each recipe + # (rprovides has to cover RPROVIDES, PACKAGES, PACKAGES_DYNAMIC) + # + msg += "\nIt could be that one recipe provides something the other doesn't and should. The following provider and runtime provider differences may be helpful." + provide_results = {} + rprovide_results = {} + commonprovs = None + commonrprovs = None + for provfn in prov_list[prov]: + provides = set(self.dataCache.fn_provides[provfn]) + rprovides = set() + for rprovide in self.dataCache.rproviders: + if provfn in self.dataCache.rproviders[rprovide]: + rprovides.add(rprovide) + for package in self.dataCache.packages: + if provfn in self.dataCache.packages[package]: + rprovides.add(package) + for package in self.dataCache.packages_dynamic: + if provfn in self.dataCache.packages_dynamic[package]: + rprovides.add(package) + if not commonprovs: + commonprovs = set(provides) + else: + commonprovs &= provides + provide_results[provfn] = provides + if not commonrprovs: + commonrprovs = set(rprovides) + else: + commonrprovs &= rprovides + rprovide_results[provfn] = rprovides + #msg += "\nCommon provides:\n %s" % ("\n ".join(commonprovs)) + #msg += "\nCommon rprovides:\n %s" % ("\n ".join(commonrprovs)) + for provfn in prov_list[prov]: + msg += "\n%s has unique provides:\n %s" % (provfn, "\n ".join(provide_results[provfn] - commonprovs)) + msg += "\n%s has unique rprovides:\n %s" % (provfn, "\n ".join(rprovide_results[provfn] - commonrprovs)) + + if self.warn_multi_bb: + logger.warn(msg) + else: + logger.error(msg) + + # Create a whitelist usable by the stamp checks + stampfnwhitelist = [] + for entry in self.stampwhitelist.split(): + entryid = self.taskData.getbuild_id(entry) + if entryid not in self.taskData.build_targets: + continue + fnid = self.taskData.build_targets[entryid][0] + fn = self.taskData.fn_index[fnid] + stampfnwhitelist.append(fn) + self.stampfnwhitelist = stampfnwhitelist + + # Iterate over the task list looking for tasks with a 'setscene' function + self.runq_setscene = [] + if not self.cooker.configuration.nosetscene: + for task in range(len(self.runq_fnid)): + setscene = taskData.gettask_id(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task] + "_setscene", False) + if not setscene: + continue + self.runq_setscene.append(task) + + def invalidate_task(fn, taskname, error_nostamp): + taskdep = self.dataCache.task_deps[fn] + fnid = self.taskData.getfn_id(fn) + if taskname not in taskData.tasks_lookup[fnid]: + logger.warn("Task %s does not exist, invalidating this task will have no effect" % taskname) + if 'nostamp' in taskdep and taskname in taskdep['nostamp']: + if error_nostamp: + bb.fatal("Task %s is marked nostamp, cannot invalidate this task" % taskname) + else: + bb.debug(1, "Task %s is marked nostamp, cannot invalidate this task" % taskname) + else: + logger.verbose("Invalidate task %s, %s", taskname, fn) + bb.parse.siggen.invalidate_task(taskname, self.dataCache, fn) + + # Invalidate task if force mode active + if self.cooker.configuration.force: + for (fn, target) in self.target_pairs: + invalidate_task(fn, target, False) + + # Invalidate task if invalidate mode active + if self.cooker.configuration.invalidate_stamp: + for (fn, target) in self.target_pairs: + for st in self.cooker.configuration.invalidate_stamp.split(','): + if not st.startswith("do_"): + st = "do_%s" % st + invalidate_task(fn, st, True) + + # Create and print to the logs a virtual/xxxx -> PN (fn) table + virtmap = taskData.get_providermap(prefix="virtual/") + virtpnmap = {} + for v in virtmap: + virtpnmap[v] = self.dataCache.pkg_fn[virtmap[v]] + bb.debug(2, "%s resolved to: %s (%s)" % (v, virtpnmap[v], virtmap[v])) + if hasattr(bb.parse.siggen, "tasks_resolved"): + bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCache) + + # Iterate over the task list and call into the siggen code + dealtwith = set() + todeal = set(range(len(self.runq_fnid))) + while len(todeal) > 0: + for task in todeal.copy(): + if len(self.runq_depends[task] - dealtwith) == 0: + dealtwith.add(task) + todeal.remove(task) + procdep = [] + for dep in self.runq_depends[task]: + procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep]) + self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache) + + bb.parse.siggen.writeout_file_checksum_cache() + return len(self.runq_fnid) + + def dump_data(self, taskQueue): + """ + Dump some debug information on the internal data structures + """ + logger.debug(3, "run_tasks:") + for task in xrange(len(self.rqdata.runq_task)): + logger.debug(3, " (%s)%s - %s: %s Deps %s RevDeps %s", task, + taskQueue.fn_index[self.rqdata.runq_fnid[task]], + self.rqdata.runq_task[task], + self.rqdata.runq_weight[task], + self.rqdata.runq_depends[task], + self.rqdata.runq_revdeps[task]) + + logger.debug(3, "sorted_tasks:") + for task1 in xrange(len(self.rqdata.runq_task)): + if task1 in self.prio_map: + task = self.prio_map[task1] + logger.debug(3, " (%s)%s - %s: %s Deps %s RevDeps %s", task, + taskQueue.fn_index[self.rqdata.runq_fnid[task]], + self.rqdata.runq_task[task], + self.rqdata.runq_weight[task], + self.rqdata.runq_depends[task], + self.rqdata.runq_revdeps[task]) + +class RunQueue: + def __init__(self, cooker, cfgData, dataCache, taskData, targets): + + self.cooker = cooker + self.cfgData = cfgData + self.rqdata = RunQueueData(self, cooker, cfgData, dataCache, taskData, targets) + + self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY", True) or "perfile" + self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION", True) or None + self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION", True) or None + self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID", True) or None + + self.state = runQueuePrepare + + # For disk space monitor + self.dm = monitordisk.diskMonitor(cfgData) + + self.rqexe = None + self.worker = None + self.workerpipe = None + self.fakeworker = None + self.fakeworkerpipe = None + + def _start_worker(self, fakeroot = False, rqexec = None): + logger.debug(1, "Starting bitbake-worker") + magic = "decafbad" + if self.cooker.configuration.profile: + magic = "decafbadbad" + if fakeroot: + magic = magic + "beef" + fakerootcmd = self.cfgData.getVar("FAKEROOTCMD", True) + fakerootenv = (self.cfgData.getVar("FAKEROOTBASEENV", True) or "").split() + env = os.environ.copy() + for key, value in (var.split('=') for var in fakerootenv): + env[key] = value + worker = subprocess.Popen([fakerootcmd, "bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env) + else: + worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE) + bb.utils.nonblockingfd(worker.stdout) + workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec) + + workerdata = { + "taskdeps" : self.rqdata.dataCache.task_deps, + "fakerootenv" : self.rqdata.dataCache.fakerootenv, + "fakerootdirs" : self.rqdata.dataCache.fakerootdirs, + "fakerootnoenv" : self.rqdata.dataCache.fakerootnoenv, + "sigdata" : bb.parse.siggen.get_taskdata(), + "runq_hash" : self.rqdata.runq_hash, + "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel, + "logdefaultverbose" : bb.msg.loggerDefaultVerbose, + "logdefaultverboselogs" : bb.msg.loggerVerboseLogs, + "logdefaultdomain" : bb.msg.loggerDefaultDomains, + "prhost" : self.cooker.prhost, + "buildname" : self.cfgData.getVar("BUILDNAME", True), + "date" : self.cfgData.getVar("DATE", True), + "time" : self.cfgData.getVar("TIME", True), + } + + worker.stdin.write("<cookerconfig>" + pickle.dumps(self.cooker.configuration) + "</cookerconfig>") + worker.stdin.write("<workerdata>" + pickle.dumps(workerdata) + "</workerdata>") + worker.stdin.flush() + + return worker, workerpipe + + def _teardown_worker(self, worker, workerpipe): + if not worker: + return + logger.debug(1, "Teardown for bitbake-worker") + try: + worker.stdin.write("<quit></quit>") + worker.stdin.flush() + except IOError: + pass + while worker.returncode is None: + workerpipe.read() + worker.poll() + while workerpipe.read(): + continue + workerpipe.close() + + def start_worker(self): + if self.worker: + self.teardown_workers() + self.teardown = False + self.worker, self.workerpipe = self._start_worker() + + def start_fakeworker(self, rqexec): + if not self.fakeworker: + self.fakeworker, self.fakeworkerpipe = self._start_worker(True, rqexec) + + def teardown_workers(self): + self.teardown = True + self._teardown_worker(self.worker, self.workerpipe) + self.worker = None + self.workerpipe = None + self._teardown_worker(self.fakeworker, self.fakeworkerpipe) + self.fakeworker = None + self.fakeworkerpipe = None + + def read_workers(self): + self.workerpipe.read() + if self.fakeworkerpipe: + self.fakeworkerpipe.read() + + def active_fds(self): + fds = [] + if self.workerpipe: + fds.append(self.workerpipe.input) + if self.fakeworkerpipe: + fds.append(self.fakeworkerpipe.input) + return fds + + def check_stamp_task(self, task, taskname = None, recurse = False, cache = None): + def get_timestamp(f): + try: + if not os.access(f, os.F_OK): + return None + return os.stat(f)[stat.ST_MTIME] + except: + return None + + if self.stamppolicy == "perfile": + fulldeptree = False + else: + fulldeptree = True + stampwhitelist = [] + if self.stamppolicy == "whitelist": + stampwhitelist = self.rqdata.stampfnwhitelist + + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] + if taskname is None: + taskname = self.rqdata.runq_task[task] + + stampfile = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) + + # If the stamp is missing, it's not current + if not os.access(stampfile, os.F_OK): + logger.debug(2, "Stampfile %s not available", stampfile) + return False + # If it's a 'nostamp' task, it's not current + taskdep = self.rqdata.dataCache.task_deps[fn] + if 'nostamp' in taskdep and taskname in taskdep['nostamp']: + logger.debug(2, "%s.%s is nostamp\n", fn, taskname) + return False + + if taskname != "do_setscene" and taskname.endswith("_setscene"): + return True + + if cache is None: + cache = {} + + iscurrent = True + t1 = get_timestamp(stampfile) + for dep in self.rqdata.runq_depends[task]: + if iscurrent: + fn2 = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[dep]] + taskname2 = self.rqdata.runq_task[dep] + stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCache, fn2) + stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCache, fn2) + t2 = get_timestamp(stampfile2) + t3 = get_timestamp(stampfile3) + if t3 and t3 > t2: + continue + if fn == fn2 or (fulldeptree and fn2 not in stampwhitelist): + if not t2: + logger.debug(2, 'Stampfile %s does not exist', stampfile2) + iscurrent = False + if t1 < t2: + logger.debug(2, 'Stampfile %s < %s', stampfile, stampfile2) + iscurrent = False + if recurse and iscurrent: + if dep in cache: + iscurrent = cache[dep] + if not iscurrent: + logger.debug(2, 'Stampfile for dependency %s:%s invalid (cached)' % (fn2, taskname2)) + else: + iscurrent = self.check_stamp_task(dep, recurse=True, cache=cache) + cache[dep] = iscurrent + if recurse: + cache[task] = iscurrent + return iscurrent + + def _execute_runqueue(self): + """ + Run the tasks in a queue prepared by rqdata.prepare() + Upon failure, optionally try to recover the build using any alternate providers + (if the abort on failure configuration option isn't set) + """ + + retval = True + + if self.state is runQueuePrepare: + self.rqexe = RunQueueExecuteDummy(self) + if self.rqdata.prepare() == 0: + self.state = runQueueComplete + else: + self.state = runQueueSceneInit + + # we are ready to run, see if any UI client needs the dependency info + if bb.cooker.CookerFeatures.SEND_DEPENDS_TREE in self.cooker.featureset: + depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData) + bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data) + + if self.state is runQueueSceneInit: + dump = self.cooker.configuration.dump_signatures + if dump: + if 'printdiff' in dump: + invalidtasks = self.print_diffscenetasks() + self.dump_signatures(dump) + if 'printdiff' in dump: + self.write_diffscenetasks(invalidtasks) + self.state = runQueueComplete + else: + self.start_worker() + self.rqexe = RunQueueExecuteScenequeue(self) + + if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]: + self.dm.check(self) + + if self.state is runQueueSceneRun: + retval = self.rqexe.execute() + + if self.state is runQueueRunInit: + if self.cooker.configuration.setsceneonly: + self.state = runQueueComplete + else: + logger.info("Executing RunQueue Tasks") + self.rqexe = RunQueueExecuteTasks(self) + self.state = runQueueRunning + + if self.state is runQueueRunning: + retval = self.rqexe.execute() + + if self.state is runQueueCleanUp: + retval = self.rqexe.finish() + + if (self.state is runQueueComplete or self.state is runQueueFailed) and self.rqexe: + self.teardown_workers() + if self.rqexe.stats.failed: + logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed.", self.rqexe.stats.completed + self.rqexe.stats.failed, self.rqexe.stats.skipped, self.rqexe.stats.failed) + else: + # Let's avoid the word "failed" if nothing actually did + logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and all succeeded.", self.rqexe.stats.completed, self.rqexe.stats.skipped) + + if self.state is runQueueFailed: + if not self.rqdata.taskData.tryaltconfigs: + raise bb.runqueue.TaskFailure(self.rqexe.failed_fnids) + for fnid in self.rqexe.failed_fnids: + self.rqdata.taskData.fail_fnid(fnid) + self.rqdata.reset() + + if self.state is runQueueComplete: + # All done + return False + + # Loop + return retval + + def execute_runqueue(self): + # Catch unexpected exceptions and ensure we exit when an error occurs, not loop. + try: + return self._execute_runqueue() + except bb.runqueue.TaskFailure: + raise + except SystemExit: + raise + except bb.BBHandledException: + try: + self.teardown_workers() + except: + pass + self.state = runQueueComplete + raise + except: + logger.error("An uncaught exception occured in runqueue, please see the failure below:") + try: + self.teardown_workers() + except: + pass + self.state = runQueueComplete + raise + + def finish_runqueue(self, now = False): + if not self.rqexe: + self.state = runQueueComplete + return + + if now: + self.rqexe.finish_now() + else: + self.rqexe.finish() + + def dump_signatures(self, options): + done = set() + bb.note("Reparsing files to collect dependency data") + for task in range(len(self.rqdata.runq_fnid)): + if self.rqdata.runq_fnid[task] not in done: + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] + the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn), self.cooker.data) + done.add(self.rqdata.runq_fnid[task]) + + bb.parse.siggen.dump_sigs(self.rqdata.dataCache, options) + + return + + def print_diffscenetasks(self): + + valid = [] + sq_hash = [] + sq_hashfn = [] + sq_fn = [] + sq_taskname = [] + sq_task = [] + noexec = [] + stamppresent = [] + valid_new = set() + + for task in xrange(len(self.rqdata.runq_fnid)): + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] + taskname = self.rqdata.runq_task[task] + taskdep = self.rqdata.dataCache.task_deps[fn] + + if 'noexec' in taskdep and taskname in taskdep['noexec']: + noexec.append(task) + continue + + sq_fn.append(fn) + sq_hashfn.append(self.rqdata.dataCache.hashfn[fn]) + sq_hash.append(self.rqdata.runq_hash[task]) + sq_taskname.append(taskname) + sq_task.append(task) + locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data } + try: + call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=True)" + valid = bb.utils.better_eval(call, locs) + # Handle version with no siginfo parameter + except TypeError: + call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)" + valid = bb.utils.better_eval(call, locs) + for v in valid: + valid_new.add(sq_task[v]) + + # Tasks which are both setscene and noexec never care about dependencies + # We therefore find tasks which are setscene and noexec and mark their + # unique dependencies as valid. + for task in noexec: + if task not in self.rqdata.runq_setscene: + continue + for dep in self.rqdata.runq_depends[task]: + hasnoexecparents = True + for dep2 in self.rqdata.runq_revdeps[dep]: + if dep2 in self.rqdata.runq_setscene and dep2 in noexec: + continue + hasnoexecparents = False + break + if hasnoexecparents: + valid_new.add(dep) + + invalidtasks = set() + for task in xrange(len(self.rqdata.runq_fnid)): + if task not in valid_new and task not in noexec: + invalidtasks.add(task) + + found = set() + processed = set() + for task in invalidtasks: + toprocess = set([task]) + while toprocess: + next = set() + for t in toprocess: + for dep in self.rqdata.runq_depends[t]: + if dep in invalidtasks: + found.add(task) + if dep not in processed: + processed.add(dep) + next.add(dep) + toprocess = next + if task in found: + toprocess = set() + + tasklist = [] + for task in invalidtasks.difference(found): + tasklist.append(self.rqdata.get_user_idstring(task)) + + if tasklist: + bb.plain("The differences between the current build and any cached tasks start at the following tasks:\n" + "\n".join(tasklist)) + + return invalidtasks.difference(found) + + def write_diffscenetasks(self, invalidtasks): + + # Define recursion callback + def recursecb(key, hash1, hash2): + hashes = [hash1, hash2] + hashfiles = bb.siggen.find_siginfo(key, None, hashes, self.cfgData) + + recout = [] + if len(hashfiles) == 2: + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb) + recout.extend(list(' ' + l for l in out2)) + else: + recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2)) + + return recout + + + for task in invalidtasks: + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] + pn = self.rqdata.dataCache.pkg_fn[fn] + taskname = self.rqdata.runq_task[task] + h = self.rqdata.runq_hash[task] + matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData) + match = None + for m in matches: + if h in m: + match = m + if match is None: + bb.fatal("Can't find a task we're supposed to have written out? (hash: %s)?" % h) + matches = {k : v for k, v in matches.iteritems() if h not in k} + if matches: + latestmatch = sorted(matches.keys(), key=lambda f: matches[f])[-1] + prevh = __find_md5__.search(latestmatch).group(0) + output = bb.siggen.compare_sigfiles(latestmatch, match, recursecb) + bb.plain("\nTask %s:%s couldn't be used from the cache because:\n We need hash %s, closest matching task was %s\n " % (pn, taskname, h, prevh) + '\n '.join(output)) + +class RunQueueExecute: + + def __init__(self, rq): + self.rq = rq + self.cooker = rq.cooker + self.cfgData = rq.cfgData + self.rqdata = rq.rqdata + + self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS", True) or 1) + self.scheduler = self.cfgData.getVar("BB_SCHEDULER", True) or "speed" + + self.runq_buildable = [] + self.runq_running = [] + self.runq_complete = [] + + self.build_stamps = {} + self.build_stamps2 = [] + self.failed_fnids = [] + + self.stampcache = {} + + rq.workerpipe.setrunqueueexec(self) + if rq.fakeworkerpipe: + rq.fakeworkerpipe.setrunqueueexec(self) + + if self.number_tasks <= 0: + bb.fatal("Invalid BB_NUMBER_THREADS %s" % self.number_tasks) + + def runqueue_process_waitpid(self, task, status): + + # self.build_stamps[pid] may not exist when use shared work directory. + if task in self.build_stamps: + self.build_stamps2.remove(self.build_stamps[task]) + del self.build_stamps[task] + + if status != 0: + self.task_fail(task, status) + else: + self.task_complete(task) + return True + + def finish_now(self): + + for worker in [self.rq.worker, self.rq.fakeworker]: + if not worker: + continue + try: + worker.stdin.write("<finishnow></finishnow>") + worker.stdin.flush() + except IOError: + # worker must have died? + pass + + if len(self.failed_fnids) != 0: + self.rq.state = runQueueFailed + return + + self.rq.state = runQueueComplete + return + + def finish(self): + self.rq.state = runQueueCleanUp + + if self.stats.active > 0: + bb.event.fire(runQueueExitWait(self.stats.active), self.cfgData) + self.rq.read_workers() + return self.rq.active_fds() + + if len(self.failed_fnids) != 0: + self.rq.state = runQueueFailed + return True + + self.rq.state = runQueueComplete + return True + + def check_dependencies(self, task, taskdeps, setscene = False): + if not self.rq.depvalidate: + return False + + taskdata = {} + taskdeps.add(task) + for dep in taskdeps: + if setscene: + depid = self.rqdata.runq_setscene[dep] + else: + depid = dep + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[depid]] + pn = self.rqdata.dataCache.pkg_fn[fn] + taskname = self.rqdata.runq_task[depid] + taskdata[dep] = [pn, taskname, fn] + call = self.rq.depvalidate + "(task, taskdata, notneeded, d)" + locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.expanded_data } + valid = bb.utils.better_eval(call, locs) + return valid + +class RunQueueExecuteDummy(RunQueueExecute): + def __init__(self, rq): + self.rq = rq + self.stats = RunQueueStats(0) + + def finish(self): + self.rq.state = runQueueComplete + return + +class RunQueueExecuteTasks(RunQueueExecute): + def __init__(self, rq): + RunQueueExecute.__init__(self, rq) + + self.stats = RunQueueStats(len(self.rqdata.runq_fnid)) + + self.stampcache = {} + + initial_covered = self.rq.scenequeue_covered.copy() + + # Mark initial buildable tasks + for task in xrange(self.stats.total): + self.runq_running.append(0) + self.runq_complete.append(0) + if len(self.rqdata.runq_depends[task]) == 0: + self.runq_buildable.append(1) + else: + self.runq_buildable.append(0) + if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered): + self.rq.scenequeue_covered.add(task) + + found = True + while found: + found = False + for task in xrange(self.stats.total): + if task in self.rq.scenequeue_covered: + continue + logger.debug(1, 'Considering %s (%s): %s' % (task, self.rqdata.get_user_idstring(task), str(self.rqdata.runq_revdeps[task]))) + + if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered): + found = True + self.rq.scenequeue_covered.add(task) + + logger.debug(1, 'Skip list (pre setsceneverify) %s', sorted(self.rq.scenequeue_covered)) + + # Allow the metadata to elect for setscene tasks to run anyway + covered_remove = set() + if self.rq.setsceneverify: + invalidtasks = [] + for task in xrange(len(self.rqdata.runq_task)): + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] + taskname = self.rqdata.runq_task[task] + taskdep = self.rqdata.dataCache.task_deps[fn] + + if 'noexec' in taskdep and taskname in taskdep['noexec']: + continue + if self.rq.check_stamp_task(task, taskname + "_setscene", cache=self.stampcache): + logger.debug(2, 'Setscene stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(task)) + continue + if self.rq.check_stamp_task(task, taskname, recurse = True, cache=self.stampcache): + logger.debug(2, 'Normal stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(task)) + continue + invalidtasks.append(task) + + call = self.rq.setsceneverify + "(covered, tasknames, fnids, fns, d, invalidtasks=invalidtasks)" + call2 = self.rq.setsceneverify + "(covered, tasknames, fnids, fns, d)" + locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : self.rqdata.runq_task, "fnids" : self.rqdata.runq_fnid, "fns" : self.rqdata.taskData.fn_index, "d" : self.cooker.expanded_data, "invalidtasks" : invalidtasks } + # Backwards compatibility with older versions without invalidtasks + try: + covered_remove = bb.utils.better_eval(call, locs) + except TypeError: + covered_remove = bb.utils.better_eval(call2, locs) + + def removecoveredtask(task): + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] + taskname = self.rqdata.runq_task[task] + '_setscene' + bb.build.del_stamp(taskname, self.rqdata.dataCache, fn) + self.rq.scenequeue_covered.remove(task) + + toremove = covered_remove + for task in toremove: + logger.debug(1, 'Not skipping task %s due to setsceneverify', task) + while toremove: + covered_remove = [] + for task in toremove: + removecoveredtask(task) + for deptask in self.rqdata.runq_depends[task]: + if deptask not in self.rq.scenequeue_covered: + continue + if deptask in toremove or deptask in covered_remove or deptask in initial_covered: + continue + logger.debug(1, 'Task %s depends on task %s so not skipping' % (task, deptask)) + covered_remove.append(deptask) + toremove = covered_remove + + logger.debug(1, 'Full skip list %s', self.rq.scenequeue_covered) + + event.fire(bb.event.StampUpdate(self.rqdata.target_pairs, self.rqdata.dataCache.stamp), self.cfgData) + + schedulers = self.get_schedulers() + for scheduler in schedulers: + if self.scheduler == scheduler.name: + self.sched = scheduler(self, self.rqdata) + logger.debug(1, "Using runqueue scheduler '%s'", scheduler.name) + break + else: + bb.fatal("Invalid scheduler '%s'. Available schedulers: %s" % + (self.scheduler, ", ".join(obj.name for obj in schedulers))) + + def get_schedulers(self): + schedulers = set(obj for obj in globals().values() + if type(obj) is type and + issubclass(obj, RunQueueScheduler)) + + user_schedulers = self.cfgData.getVar("BB_SCHEDULERS", True) + if user_schedulers: + for sched in user_schedulers.split(): + if not "." in sched: + bb.note("Ignoring scheduler '%s' from BB_SCHEDULERS: not an import" % sched) + continue + + modname, name = sched.rsplit(".", 1) + try: + module = __import__(modname, fromlist=(name,)) + except ImportError as exc: + logger.critical("Unable to import scheduler '%s' from '%s': %s" % (name, modname, exc)) + raise SystemExit(1) + else: + schedulers.add(getattr(module, name)) + return schedulers + + def setbuildable(self, task): + self.runq_buildable[task] = 1 + self.sched.newbuilable(task) + + def task_completeoutright(self, task): + """ + Mark a task as completed + Look at the reverse dependencies and mark any task with + completed dependencies as buildable + """ + self.runq_complete[task] = 1 + for revdep in self.rqdata.runq_revdeps[task]: + if self.runq_running[revdep] == 1: + continue + if self.runq_buildable[revdep] == 1: + continue + alldeps = 1 + for dep in self.rqdata.runq_depends[revdep]: + if self.runq_complete[dep] != 1: + alldeps = 0 + if alldeps == 1: + self.setbuildable(revdep) + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[revdep]] + taskname = self.rqdata.runq_task[revdep] + logger.debug(1, "Marking task %s (%s, %s) as buildable", revdep, fn, taskname) + + def task_complete(self, task): + self.stats.taskCompleted() + bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) + self.task_completeoutright(task) + + def task_fail(self, task, exitcode): + """ + Called when a task has failed + Updates the state engine with the failure + """ + self.stats.taskFailed() + fnid = self.rqdata.runq_fnid[task] + self.failed_fnids.append(fnid) + bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData) + if self.rqdata.taskData.abort: + self.rq.state = runQueueCleanUp + + def task_skip(self, task, reason): + self.runq_running[task] = 1 + self.setbuildable(task) + bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData) + self.task_completeoutright(task) + self.stats.taskCompleted() + self.stats.taskSkipped() + + def execute(self): + """ + Run the tasks in a queue prepared by rqdata.prepare() + """ + + self.rq.read_workers() + + + if self.stats.total == 0: + # nothing to do + self.rq.state = runQueueCleanUp + + task = self.sched.next() + if task is not None: + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] + taskname = self.rqdata.runq_task[task] + + if task in self.rq.scenequeue_covered: + logger.debug(2, "Setscene covered task %s (%s)", task, + self.rqdata.get_user_idstring(task)) + self.task_skip(task, "covered") + return True + + if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): + logger.debug(2, "Stamp current task %s (%s)", task, + self.rqdata.get_user_idstring(task)) + self.task_skip(task, "existing") + return True + + taskdep = self.rqdata.dataCache.task_deps[fn] + if 'noexec' in taskdep and taskname in taskdep['noexec']: + startevent = runQueueTaskStarted(task, self.stats, self.rq, + noexec=True) + bb.event.fire(startevent, self.cfgData) + self.runq_running[task] = 1 + self.stats.taskActive() + if not self.cooker.configuration.dry_run: + bb.build.make_stamp(taskname, self.rqdata.dataCache, fn) + self.task_complete(task) + return True + else: + startevent = runQueueTaskStarted(task, self.stats, self.rq) + bb.event.fire(startevent, self.cfgData) + + taskdepdata = self.build_taskdepdata(task) + + taskdep = self.rqdata.dataCache.task_deps[fn] + if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: + if not self.rq.fakeworker: + try: + self.rq.start_fakeworker(self) + except OSError as exc: + logger.critical("Failed to spawn fakeroot worker to run %s:%s: %s" % (fn, taskname, str(exc))) + self.rq.state = runQueueFailed + return True + self.rq.fakeworker.stdin.write("<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + "</runtask>") + self.rq.fakeworker.stdin.flush() + else: + self.rq.worker.stdin.write("<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + "</runtask>") + self.rq.worker.stdin.flush() + + self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) + self.build_stamps2.append(self.build_stamps[task]) + self.runq_running[task] = 1 + self.stats.taskActive() + if self.stats.active < self.number_tasks: + return True + + if self.stats.active > 0: + self.rq.read_workers() + return self.rq.active_fds() + + if len(self.failed_fnids) != 0: + self.rq.state = runQueueFailed + return True + + # Sanity Checks + for task in xrange(self.stats.total): + if self.runq_buildable[task] == 0: + logger.error("Task %s never buildable!", task) + if self.runq_running[task] == 0: + logger.error("Task %s never ran!", task) + if self.runq_complete[task] == 0: + logger.error("Task %s never completed!", task) + self.rq.state = runQueueComplete + + return True + + def build_taskdepdata(self, task): + taskdepdata = {} + next = self.rqdata.runq_depends[task] + next.add(task) + while next: + additional = [] + for revdep in next: + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[revdep]] + pn = self.rqdata.dataCache.pkg_fn[fn] + taskname = self.rqdata.runq_task[revdep] + deps = self.rqdata.runq_depends[revdep] + provides = self.rqdata.dataCache.fn_provides[fn] + taskdepdata[revdep] = [pn, taskname, fn, deps, provides] + for revdep2 in deps: + if revdep2 not in taskdepdata: + additional.append(revdep2) + next = additional + + #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) + return taskdepdata + +class RunQueueExecuteScenequeue(RunQueueExecute): + def __init__(self, rq): + RunQueueExecute.__init__(self, rq) + + self.scenequeue_covered = set() + self.scenequeue_notcovered = set() + self.scenequeue_notneeded = set() + + # If we don't have any setscene functions, skip this step + if len(self.rqdata.runq_setscene) == 0: + rq.scenequeue_covered = set() + rq.state = runQueueRunInit + return + + self.stats = RunQueueStats(len(self.rqdata.runq_setscene)) + + sq_revdeps = [] + sq_revdeps_new = [] + sq_revdeps_squash = [] + self.sq_harddeps = {} + + # We need to construct a dependency graph for the setscene functions. Intermediate + # dependencies between the setscene tasks only complicate the code. This code + # therefore aims to collapse the huge runqueue dependency tree into a smaller one + # only containing the setscene functions. + + for task in xrange(self.stats.total): + self.runq_running.append(0) + self.runq_complete.append(0) + self.runq_buildable.append(0) + + # First process the chains up to the first setscene task. + endpoints = {} + for task in xrange(len(self.rqdata.runq_fnid)): + sq_revdeps.append(copy.copy(self.rqdata.runq_revdeps[task])) + sq_revdeps_new.append(set()) + if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene: + endpoints[task] = set() + + # Secondly process the chains between setscene tasks. + for task in self.rqdata.runq_setscene: + for dep in self.rqdata.runq_depends[task]: + if dep not in endpoints: + endpoints[dep] = set() + endpoints[dep].add(task) + + def process_endpoints(endpoints): + newendpoints = {} + for point, task in endpoints.items(): + tasks = set() + if task: + tasks |= task + if sq_revdeps_new[point]: + tasks |= sq_revdeps_new[point] + sq_revdeps_new[point] = set() + if point in self.rqdata.runq_setscene: + sq_revdeps_new[point] = tasks + tasks = set() + for dep in self.rqdata.runq_depends[point]: + if point in sq_revdeps[dep]: + sq_revdeps[dep].remove(point) + if tasks: + sq_revdeps_new[dep] |= tasks + if (len(sq_revdeps[dep]) == 0 or len(sq_revdeps_new[dep]) != 0) and dep not in self.rqdata.runq_setscene: + newendpoints[dep] = task + if len(newendpoints) != 0: + process_endpoints(newendpoints) + + process_endpoints(endpoints) + + # Build a list of setscene tasks which are "unskippable" + # These are direct endpoints referenced by the build + endpoints2 = {} + sq_revdeps2 = [] + sq_revdeps_new2 = [] + def process_endpoints2(endpoints): + newendpoints = {} + for point, task in endpoints.items(): + tasks = set([point]) + if task: + tasks |= task + if sq_revdeps_new2[point]: + tasks |= sq_revdeps_new2[point] + sq_revdeps_new2[point] = set() + if point in self.rqdata.runq_setscene: + sq_revdeps_new2[point] = tasks + for dep in self.rqdata.runq_depends[point]: + if point in sq_revdeps2[dep]: + sq_revdeps2[dep].remove(point) + if tasks: + sq_revdeps_new2[dep] |= tasks + if (len(sq_revdeps2[dep]) == 0 or len(sq_revdeps_new2[dep]) != 0) and dep not in self.rqdata.runq_setscene: + newendpoints[dep] = tasks + if len(newendpoints) != 0: + process_endpoints2(newendpoints) + for task in xrange(len(self.rqdata.runq_fnid)): + sq_revdeps2.append(copy.copy(self.rqdata.runq_revdeps[task])) + sq_revdeps_new2.append(set()) + if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene: + endpoints2[task] = set() + process_endpoints2(endpoints2) + self.unskippable = [] + for task in self.rqdata.runq_setscene: + if sq_revdeps_new2[task]: + self.unskippable.append(self.rqdata.runq_setscene.index(task)) + + for task in xrange(len(self.rqdata.runq_fnid)): + if task in self.rqdata.runq_setscene: + deps = set() + for dep in sq_revdeps_new[task]: + deps.add(self.rqdata.runq_setscene.index(dep)) + sq_revdeps_squash.append(deps) + elif len(sq_revdeps_new[task]) != 0: + bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.") + + # Resolve setscene inter-task dependencies + # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" + # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies + for task in self.rqdata.runq_setscene: + realid = self.rqdata.taskData.gettask_id(self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]], self.rqdata.runq_task[task] + "_setscene", False) + idepends = self.rqdata.taskData.tasks_idepends[realid] + for (depid, idependtask) in idepends: + if depid not in self.rqdata.taskData.build_targets: + continue + + depdata = self.rqdata.taskData.build_targets[depid][0] + if depdata is None: + continue + dep = self.rqdata.taskData.fn_index[depdata] + taskid = self.rqdata.get_task_id(self.rqdata.taskData.getfn_id(dep), idependtask.replace("_setscene", "")) + if taskid is None: + bb.msg.fatal("RunQueue", "Task %s_setscene depends upon non-existent task %s:%s" % (self.rqdata.get_user_idstring(task), dep, idependtask)) + + if not self.rqdata.runq_setscene.index(taskid) in self.sq_harddeps: + self.sq_harddeps[self.rqdata.runq_setscene.index(taskid)] = set() + self.sq_harddeps[self.rqdata.runq_setscene.index(taskid)].add(self.rqdata.runq_setscene.index(task)) + + sq_revdeps_squash[self.rqdata.runq_setscene.index(task)].add(self.rqdata.runq_setscene.index(taskid)) + # Have to zero this to avoid circular dependencies + sq_revdeps_squash[self.rqdata.runq_setscene.index(taskid)] = set() + + for task in self.sq_harddeps: + for dep in self.sq_harddeps[task]: + sq_revdeps_squash[dep].add(task) + + #for task in xrange(len(sq_revdeps_squash)): + # realtask = self.rqdata.runq_setscene[task] + # bb.warn("Task %s: %s_setscene is %s " % (task, self.rqdata.get_user_idstring(realtask) , sq_revdeps_squash[task])) + + self.sq_deps = [] + self.sq_revdeps = sq_revdeps_squash + self.sq_revdeps2 = copy.deepcopy(self.sq_revdeps) + + for task in xrange(len(self.sq_revdeps)): + self.sq_deps.append(set()) + for task in xrange(len(self.sq_revdeps)): + for dep in self.sq_revdeps[task]: + self.sq_deps[dep].add(task) + + for task in xrange(len(self.sq_revdeps)): + if len(self.sq_revdeps[task]) == 0: + self.runq_buildable[task] = 1 + + self.outrightfail = [] + if self.rq.hashvalidate: + sq_hash = [] + sq_hashfn = [] + sq_fn = [] + sq_taskname = [] + sq_task = [] + noexec = [] + stamppresent = [] + for task in xrange(len(self.sq_revdeps)): + realtask = self.rqdata.runq_setscene[task] + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]] + taskname = self.rqdata.runq_task[realtask] + taskdep = self.rqdata.dataCache.task_deps[fn] + + if 'noexec' in taskdep and taskname in taskdep['noexec']: + noexec.append(task) + self.task_skip(task) + bb.build.make_stamp(taskname + "_setscene", self.rqdata.dataCache, fn) + continue + + if self.rq.check_stamp_task(realtask, taskname + "_setscene", cache=self.stampcache): + logger.debug(2, 'Setscene stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(realtask)) + stamppresent.append(task) + self.task_skip(task) + continue + + if self.rq.check_stamp_task(realtask, taskname, recurse = True, cache=self.stampcache): + logger.debug(2, 'Normal stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(realtask)) + stamppresent.append(task) + self.task_skip(task) + continue + + sq_fn.append(fn) + sq_hashfn.append(self.rqdata.dataCache.hashfn[fn]) + sq_hash.append(self.rqdata.runq_hash[realtask]) + sq_taskname.append(taskname) + sq_task.append(task) + call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)" + locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data } + valid = bb.utils.better_eval(call, locs) + + valid_new = stamppresent + for v in valid: + valid_new.append(sq_task[v]) + + for task in xrange(len(self.sq_revdeps)): + if task not in valid_new and task not in noexec: + realtask = self.rqdata.runq_setscene[task] + logger.debug(2, 'No package found, so skipping setscene task %s', + self.rqdata.get_user_idstring(realtask)) + self.outrightfail.append(task) + + logger.info('Executing SetScene Tasks') + + self.rq.state = runQueueSceneRun + + def scenequeue_updatecounters(self, task, fail = False): + for dep in self.sq_deps[task]: + if fail and task in self.sq_harddeps and dep in self.sq_harddeps[task]: + realtask = self.rqdata.runq_setscene[task] + realdep = self.rqdata.runq_setscene[dep] + logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (self.rqdata.get_user_idstring(realtask), self.rqdata.get_user_idstring(realdep))) + self.scenequeue_updatecounters(dep, fail) + continue + if task not in self.sq_revdeps2[dep]: + # May already have been removed by the fail case above + continue + self.sq_revdeps2[dep].remove(task) + if len(self.sq_revdeps2[dep]) == 0: + self.runq_buildable[dep] = 1 + + def task_completeoutright(self, task): + """ + Mark a task as completed + Look at the reverse dependencies and mark any task with + completed dependencies as buildable + """ + + index = self.rqdata.runq_setscene[task] + logger.debug(1, 'Found task %s which could be accelerated', + self.rqdata.get_user_idstring(index)) + + self.scenequeue_covered.add(task) + self.scenequeue_updatecounters(task) + + def task_complete(self, task): + self.stats.taskCompleted() + bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) + self.task_completeoutright(task) + + def task_fail(self, task, result): + self.stats.taskFailed() + bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData) + self.scenequeue_notcovered.add(task) + self.scenequeue_updatecounters(task, True) + + def task_failoutright(self, task): + self.runq_running[task] = 1 + self.runq_buildable[task] = 1 + self.stats.taskCompleted() + self.stats.taskSkipped() + index = self.rqdata.runq_setscene[task] + self.scenequeue_notcovered.add(task) + self.scenequeue_updatecounters(task, True) + + def task_skip(self, task): + self.runq_running[task] = 1 + self.runq_buildable[task] = 1 + self.task_completeoutright(task) + self.stats.taskCompleted() + self.stats.taskSkipped() + + def execute(self): + """ + Run the tasks in a queue prepared by prepare_runqueue + """ + + self.rq.read_workers() + + task = None + if self.stats.active < self.number_tasks: + # Find the next setscene to run + for nexttask in xrange(self.stats.total): + if self.runq_buildable[nexttask] == 1 and self.runq_running[nexttask] != 1: + if nexttask in self.unskippable: + logger.debug(2, "Setscene task %s is unskippable" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask])) + if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True): + realtask = self.rqdata.runq_setscene[nexttask] + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]] + foundtarget = False + for target in self.rqdata.target_pairs: + if target[0] == fn and target[1] == self.rqdata.runq_task[realtask]: + foundtarget = True + break + if not foundtarget: + logger.debug(2, "Skipping setscene for task %s" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask])) + self.task_skip(nexttask) + self.scenequeue_notneeded.add(nexttask) + return True + if nexttask in self.outrightfail: + self.task_failoutright(nexttask) + return True + task = nexttask + break + if task is not None: + realtask = self.rqdata.runq_setscene[task] + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]] + + taskname = self.rqdata.runq_task[realtask] + "_setscene" + if self.rq.check_stamp_task(realtask, self.rqdata.runq_task[realtask], recurse = True, cache=self.stampcache): + logger.debug(2, 'Stamp for underlying task %s(%s) is current, so skipping setscene variant', + task, self.rqdata.get_user_idstring(realtask)) + self.task_failoutright(task) + return True + + if self.cooker.configuration.force: + for target in self.rqdata.target_pairs: + if target[0] == fn and target[1] == self.rqdata.runq_task[realtask]: + self.task_failoutright(task) + return True + + if self.rq.check_stamp_task(realtask, taskname, cache=self.stampcache): + logger.debug(2, 'Setscene stamp current task %s(%s), so skip it and its dependencies', + task, self.rqdata.get_user_idstring(realtask)) + self.task_skip(task) + return True + + startevent = sceneQueueTaskStarted(task, self.stats, self.rq) + bb.event.fire(startevent, self.cfgData) + + taskdep = self.rqdata.dataCache.task_deps[fn] + if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']: + if not self.rq.fakeworker: + self.rq.start_fakeworker(self) + self.rq.fakeworker.stdin.write("<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + "</runtask>") + self.rq.fakeworker.stdin.flush() + else: + self.rq.worker.stdin.write("<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + "</runtask>") + self.rq.worker.stdin.flush() + + self.runq_running[task] = 1 + self.stats.taskActive() + if self.stats.active < self.number_tasks: + return True + + if self.stats.active > 0: + self.rq.read_workers() + return self.rq.active_fds() + + #for task in xrange(self.stats.total): + # if self.runq_running[task] != 1: + # buildable = self.runq_buildable[task] + # revdeps = self.sq_revdeps[task] + # bb.warn("Found we didn't run %s %s %s %s" % (task, buildable, str(revdeps), self.rqdata.get_user_idstring(self.rqdata.runq_setscene[task]))) + + # Convert scenequeue_covered task numbers into full taskgraph ids + oldcovered = self.scenequeue_covered + self.rq.scenequeue_covered = set() + for task in oldcovered: + self.rq.scenequeue_covered.add(self.rqdata.runq_setscene[task]) + + logger.debug(1, 'We can skip tasks %s', sorted(self.rq.scenequeue_covered)) + + self.rq.state = runQueueRunInit + + completeevent = sceneQueueComplete(self.stats, self.rq) + bb.event.fire(completeevent, self.cfgData) + + return True + + def runqueue_process_waitpid(self, task, status): + task = self.rq.rqdata.runq_setscene.index(task) + + RunQueueExecute.runqueue_process_waitpid(self, task, status) + +class TaskFailure(Exception): + """ + Exception raised when a task in a runqueue fails + """ + def __init__(self, x): + self.args = x + + +class runQueueExitWait(bb.event.Event): + """ + Event when waiting for task processes to exit + """ + + def __init__(self, remain): + self.remain = remain + self.message = "Waiting for %s active tasks to finish" % remain + bb.event.Event.__init__(self) + +class runQueueEvent(bb.event.Event): + """ + Base runQueue event class + """ + def __init__(self, task, stats, rq): + self.taskid = task + self.taskstring = rq.rqdata.get_user_idstring(task) + self.taskname = rq.rqdata.get_task_name(task) + self.taskfile = rq.rqdata.get_task_file(task) + self.taskhash = rq.rqdata.get_task_hash(task) + self.stats = stats.copy() + bb.event.Event.__init__(self) + +class sceneQueueEvent(runQueueEvent): + """ + Base sceneQueue event class + """ + def __init__(self, task, stats, rq, noexec=False): + runQueueEvent.__init__(self, task, stats, rq) + realtask = rq.rqdata.runq_setscene[task] + self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene") + self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene" + self.taskfile = rq.rqdata.get_task_file(realtask) + self.taskhash = rq.rqdata.get_task_hash(realtask) + +class runQueueTaskStarted(runQueueEvent): + """ + Event notifying a task was started + """ + def __init__(self, task, stats, rq, noexec=False): + runQueueEvent.__init__(self, task, stats, rq) + self.noexec = noexec + +class sceneQueueTaskStarted(sceneQueueEvent): + """ + Event notifying a setscene task was started + """ + def __init__(self, task, stats, rq, noexec=False): + sceneQueueEvent.__init__(self, task, stats, rq) + self.noexec = noexec + +class runQueueTaskFailed(runQueueEvent): + """ + Event notifying a task failed + """ + def __init__(self, task, stats, exitcode, rq): + runQueueEvent.__init__(self, task, stats, rq) + self.exitcode = exitcode + +class sceneQueueTaskFailed(sceneQueueEvent): + """ + Event notifying a setscene task failed + """ + def __init__(self, task, stats, exitcode, rq): + sceneQueueEvent.__init__(self, task, stats, rq) + self.exitcode = exitcode + +class sceneQueueComplete(sceneQueueEvent): + """ + Event when all the sceneQueue tasks are complete + """ + def __init__(self, stats, rq): + self.stats = stats.copy() + bb.event.Event.__init__(self) + +class runQueueTaskCompleted(runQueueEvent): + """ + Event notifying a task completed + """ + +class sceneQueueTaskCompleted(sceneQueueEvent): + """ + Event notifying a setscene task completed + """ + +class runQueueTaskSkipped(runQueueEvent): + """ + Event notifying a task was skipped + """ + def __init__(self, task, stats, rq, reason): + runQueueEvent.__init__(self, task, stats, rq) + self.reason = reason + +class runQueuePipe(): + """ + Abstraction for a pipe between a worker thread and the server + """ + def __init__(self, pipein, pipeout, d, rq, rqexec): + self.input = pipein + if pipeout: + pipeout.close() + bb.utils.nonblockingfd(self.input) + self.queue = "" + self.d = d + self.rq = rq + self.rqexec = rqexec + + def setrunqueueexec(self, rqexec): + self.rqexec = rqexec + + def read(self): + for w in [self.rq.worker, self.rq.fakeworker]: + if not w: + continue + w.poll() + if w.returncode is not None and not self.rq.teardown: + name = None + if self.rq.worker and w.pid == self.rq.worker.pid: + name = "Worker" + elif self.rq.fakeworker and w.pid == self.rq.fakeworker.pid: + name = "Fakeroot" + bb.error("%s process (%s) exited unexpectedly (%s), shutting down..." % (name, w.pid, str(w.returncode))) + self.rq.finish_runqueue(True) + + start = len(self.queue) + try: + self.queue = self.queue + self.input.read(102400) + except (OSError, IOError) as e: + if e.errno != errno.EAGAIN: + raise + end = len(self.queue) + found = True + while found and len(self.queue): + found = False + index = self.queue.find("</event>") + while index != -1 and self.queue.startswith("<event>"): + try: + event = pickle.loads(self.queue[7:index]) + except ValueError as e: + bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[7:index])) + bb.event.fire_from_worker(event, self.d) + found = True + self.queue = self.queue[index+8:] + index = self.queue.find("</event>") + index = self.queue.find("</exitcode>") + while index != -1 and self.queue.startswith("<exitcode>"): + try: + task, status = pickle.loads(self.queue[10:index]) + except ValueError as e: + bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[10:index])) + self.rqexec.runqueue_process_waitpid(task, status) + found = True + self.queue = self.queue[index+11:] + index = self.queue.find("</exitcode>") + return (end > start) + + def close(self): + while self.read(): + continue + if len(self.queue) > 0: + print("Warning, worker left partial message: %s" % self.queue) + self.input.close() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/server/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/server/__init__.py new file mode 100644 index 000000000..538a633fe --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/server/__init__.py @@ -0,0 +1,99 @@ +# +# BitBake Base Server Code +# +# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer +# Copyright (C) 2006 - 2008 Richard Purdie +# Copyright (C) 2013 Alexandru Damian +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" Base code for Bitbake server process + +Have a common base for that all Bitbake server classes ensures a consistent +approach to the interface, and minimize risks associated with code duplication. + +""" + +""" BaseImplServer() the base class for all XXServer() implementations. + + These classes contain the actual code that runs the server side, i.e. + listens for the commands and executes them. Although these implementations + contain all the data of the original bitbake command, i.e the cooker instance, + they may well run on a different process or even machine. + +""" + +class BaseImplServer(): + def __init__(self): + self._idlefuns = {} + + def addcooker(self, cooker): + self.cooker = cooker + + def register_idle_function(self, function, data): + """Register a function to be called while the server is idle""" + assert hasattr(function, '__call__') + self._idlefuns[function] = data + + + +""" BitBakeBaseServerConnection class is the common ancestor to all + BitBakeServerConnection classes. + + These classes control the remote server. The only command currently + implemented is the terminate() command. + +""" + +class BitBakeBaseServerConnection(): + def __init__(self, serverImpl): + pass + + def terminate(self): + pass + + def setupEventQueue(self): + pass + + +""" BitBakeBaseServer class is the common ancestor to all Bitbake servers + + Derive this class in order to implement a BitBakeServer which is the + controlling stub for the actual server implementation + +""" +class BitBakeBaseServer(object): + def initServer(self): + self.serverImpl = None # we ensure a runtime crash if not overloaded + self.connection = None + return + + def addcooker(self, cooker): + self.cooker = cooker + self.serverImpl.addcooker(cooker) + + def getServerIdleCB(self): + return self.serverImpl.register_idle_function + + def saveConnectionDetails(self): + return + + def detach(self): + return + + def establishConnection(self, featureset): + raise "Must redefine the %s.establishConnection()" % self.__class__.__name__ + + def endSession(self): + self.connection.terminate() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/server/process.py b/import-layers/yocto-poky/bitbake/lib/bb/server/process.py new file mode 100644 index 000000000..a3078a873 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/server/process.py @@ -0,0 +1,268 @@ +# +# BitBake Process based server. +# +# Copyright (C) 2010 Bob Foerster <robert@erafx.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + This module implements a multiprocessing.Process based server for bitbake. +""" + +import bb +import bb.event +import itertools +import logging +import multiprocessing +import os +import signal +import sys +import time +import select +from Queue import Empty +from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager + +from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer + +logger = logging.getLogger('BitBake') + +class ServerCommunicator(): + def __init__(self, connection, event_handle, server): + self.connection = connection + self.event_handle = event_handle + self.server = server + + def runCommand(self, command): + # @todo try/except + self.connection.send(command) + + if not self.server.is_alive(): + raise SystemExit + + while True: + # don't let the user ctrl-c while we're waiting for a response + try: + for idx in range(0,4): # 0, 1, 2, 3 + if self.connection.poll(5): + return self.connection.recv() + else: + bb.warn("Timeout while attempting to communicate with bitbake server") + bb.fatal("Gave up; Too many tries: timeout while attempting to communicate with bitbake server") + except KeyboardInterrupt: + pass + + def getEventHandle(self): + return self.event_handle.value + +class EventAdapter(): + """ + Adapter to wrap our event queue since the caller (bb.event) expects to + call a send() method, but our actual queue only has put() + """ + def __init__(self, queue): + self.queue = queue + + def send(self, event): + try: + self.queue.put(event) + except Exception as err: + print("EventAdapter puked: %s" % str(err)) + + +class ProcessServer(Process, BaseImplServer): + profile_filename = "profile.log" + profile_processed_filename = "profile.log.processed" + + def __init__(self, command_channel, event_queue, featurelist): + BaseImplServer.__init__(self) + Process.__init__(self) + self.command_channel = command_channel + self.event_queue = event_queue + self.event = EventAdapter(event_queue) + self.featurelist = featurelist + self.quit = False + + self.quitin, self.quitout = Pipe() + self.event_handle = multiprocessing.Value("i") + + def run(self): + for event in bb.event.ui_queue: + self.event_queue.put(event) + self.event_handle.value = bb.event.register_UIHhandler(self, True) + + bb.cooker.server_main(self.cooker, self.main) + + def main(self): + # Ignore SIGINT within the server, as all SIGINT handling is done by + # the UI and communicated to us + self.quitin.close() + signal.signal(signal.SIGINT, signal.SIG_IGN) + bb.utils.set_process_name("Cooker") + while not self.quit: + try: + if self.command_channel.poll(): + command = self.command_channel.recv() + self.runCommand(command) + if self.quitout.poll(): + self.quitout.recv() + self.quit = True + try: + self.runCommand(["stateForceShutdown"]) + except: + pass + + self.idle_commands(.1, [self.command_channel, self.quitout]) + except Exception: + logger.exception('Running command %s', command) + + self.event_queue.close() + bb.event.unregister_UIHhandler(self.event_handle.value) + self.command_channel.close() + self.cooker.shutdown(True) + self.quitout.close() + + def idle_commands(self, delay, fds=None): + nextsleep = delay + if not fds: + fds = [] + + for function, data in self._idlefuns.items(): + try: + retval = function(self, data, False) + if retval is False: + del self._idlefuns[function] + nextsleep = None + elif retval is True: + nextsleep = None + elif isinstance(retval, float): + if (retval < nextsleep): + nextsleep = retval + elif nextsleep is None: + continue + else: + fds = fds + retval + except SystemExit: + raise + except Exception as exc: + if not isinstance(exc, bb.BBHandledException): + logger.exception('Running idle function') + del self._idlefuns[function] + self.quit = True + + if nextsleep is not None: + select.select(fds,[],[],nextsleep) + + def runCommand(self, command): + """ + Run a cooker command on the server + """ + self.command_channel.send(self.cooker.command.runCommand(command)) + + def stop(self): + self.quitin.send("quit") + self.quitin.close() + +class BitBakeProcessServerConnection(BitBakeBaseServerConnection): + def __init__(self, serverImpl, ui_channel, event_queue): + self.procserver = serverImpl + self.ui_channel = ui_channel + self.event_queue = event_queue + self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle, self.procserver) + self.events = self.event_queue + self.terminated = False + + def sigterm_terminate(self): + bb.error("UI received SIGTERM") + self.terminate() + + def terminate(self): + if self.terminated: + return + self.terminated = True + def flushevents(): + while True: + try: + event = self.event_queue.get(block=False) + except (Empty, IOError): + break + if isinstance(event, logging.LogRecord): + logger.handle(event) + + signal.signal(signal.SIGINT, signal.SIG_IGN) + self.procserver.stop() + + while self.procserver.is_alive(): + flushevents() + self.procserver.join(0.1) + + self.ui_channel.close() + self.event_queue.close() + self.event_queue.setexit() + +# Wrap Queue to provide API which isn't server implementation specific +class ProcessEventQueue(multiprocessing.queues.Queue): + def __init__(self, maxsize): + multiprocessing.queues.Queue.__init__(self, maxsize) + self.exit = False + bb.utils.set_process_name("ProcessEQueue") + + def setexit(self): + self.exit = True + + def waitEvent(self, timeout): + if self.exit: + sys.exit(1) + try: + if not self.server.is_alive(): + self.setexit() + return None + return self.get(True, timeout) + except Empty: + return None + + def getEvent(self): + try: + if not self.server.is_alive(): + self.setexit() + return None + return self.get(False) + except Empty: + return None + + +class BitBakeServer(BitBakeBaseServer): + def initServer(self, single_use=True): + # establish communication channels. We use bidirectional pipes for + # ui <--> server command/response pairs + # and a queue for server -> ui event notifications + # + self.ui_channel, self.server_channel = Pipe() + self.event_queue = ProcessEventQueue(0) + self.serverImpl = ProcessServer(self.server_channel, self.event_queue, None) + self.event_queue.server = self.serverImpl + + def detach(self): + self.serverImpl.start() + return + + def establishConnection(self, featureset): + + self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue) + + _, error = self.connection.connection.runCommand(["setFeatures", featureset]) + if error: + logger.error("Unable to set the cooker to the correct featureset: %s" % error) + raise BaseException(error) + signal.signal(signal.SIGTERM, lambda i, s: self.connection.sigterm_terminate()) + return self.connection diff --git a/import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py b/import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py new file mode 100644 index 000000000..ace1cf646 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py @@ -0,0 +1,390 @@ +# +# BitBake XMLRPC Server +# +# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer +# Copyright (C) 2006 - 2008 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + This module implements an xmlrpc server for BitBake. + + Use this by deriving a class from BitBakeXMLRPCServer and then adding + methods which you want to "export" via XMLRPC. If the methods have the + prefix xmlrpc_, then registering those function will happen automatically, + if not, you need to call register_function. + + Use register_idle_function() to add a function which the xmlrpc server + calls from within server_forever when no requests are pending. Make sure + that those functions are non-blocking or else you will introduce latency + in the server's main loop. +""" + +import bb +import xmlrpclib, sys +from bb import daemonize +from bb.ui import uievent +import hashlib, time +import socket +import os, signal +import threading +try: + import cPickle as pickle +except ImportError: + import pickle + +DEBUG = False + +from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler +import inspect, select, httplib + +from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer + +class BBTransport(xmlrpclib.Transport): + def __init__(self, timeout): + self.timeout = timeout + self.connection_token = None + xmlrpclib.Transport.__init__(self) + + # Modified from default to pass timeout to HTTPConnection + def make_connection(self, host): + #return an existing connection if possible. This allows + #HTTP/1.1 keep-alive. + if self._connection and host == self._connection[0]: + return self._connection[1] + + # create a HTTP connection object from a host descriptor + chost, self._extra_headers, x509 = self.get_host_info(host) + #store the host argument along with the connection object + self._connection = host, httplib.HTTPConnection(chost, timeout=self.timeout) + return self._connection[1] + + def set_connection_token(self, token): + self.connection_token = token + + def send_content(self, h, body): + if self.connection_token: + h.putheader("Bitbake-token", self.connection_token) + xmlrpclib.Transport.send_content(self, h, body) + +def _create_server(host, port, timeout = 60): + t = BBTransport(timeout) + s = xmlrpclib.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True) + return s, t + +class BitBakeServerCommands(): + + def __init__(self, server): + self.server = server + self.has_client = False + + def registerEventHandler(self, host, port): + """ + Register a remote UI Event Handler + """ + s, t = _create_server(host, port) + + # we don't allow connections if the cooker is running + if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]): + return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.cooker.state) + + self.event_handle = bb.event.register_UIHhandler(s, True) + return self.event_handle, 'OK' + + def unregisterEventHandler(self, handlerNum): + """ + Unregister a remote UI Event Handler + """ + return bb.event.unregister_UIHhandler(handlerNum) + + def runCommand(self, command): + """ + Run a cooker command on the server + """ + return self.cooker.command.runCommand(command, self.server.readonly) + + def getEventHandle(self): + return self.event_handle + + def terminateServer(self): + """ + Trigger the server to quit + """ + self.server.quit = True + print("Server (cooker) exiting") + return + + def addClient(self): + if self.has_client: + return None + token = hashlib.md5(str(time.time())).hexdigest() + self.server.set_connection_token(token) + self.has_client = True + return token + + def removeClient(self): + if self.has_client: + self.server.set_connection_token(None) + self.has_client = False + if self.server.single_use: + self.server.quit = True + +# This request handler checks if the request has a "Bitbake-token" header +# field (this comes from the client side) and compares it with its internal +# "Bitbake-token" field (this comes from the server). If the two are not +# equal, it is assumed that a client is trying to connect to the server +# while another client is connected to the server. In this case, a 503 error +# ("service unavailable") is returned to the client. +class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): + def __init__(self, request, client_address, server): + self.server = server + SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server) + + def do_POST(self): + try: + remote_token = self.headers["Bitbake-token"] + except: + remote_token = None + if remote_token != self.server.connection_token and remote_token != "observer": + self.report_503() + else: + if remote_token == "observer": + self.server.readonly = True + else: + self.server.readonly = False + SimpleXMLRPCRequestHandler.do_POST(self) + + def report_503(self): + self.send_response(503) + response = 'No more client allowed' + self.send_header("Content-type", "text/plain") + self.send_header("Content-length", str(len(response))) + self.end_headers() + self.wfile.write(response) + + +class XMLRPCProxyServer(BaseImplServer): + """ not a real working server, but a stub for a proxy server connection + + """ + def __init__(self, host, port): + self.host = host + self.port = port + +class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer): + # remove this when you're done with debugging + # allow_reuse_address = True + + def __init__(self, interface, single_use=False): + """ + Constructor + """ + BaseImplServer.__init__(self) + self.single_use = single_use + # Use auto port configuration + if (interface[1] == -1): + interface = (interface[0], 0) + SimpleXMLRPCServer.__init__(self, interface, + requestHandler=BitBakeXMLRPCRequestHandler, + logRequests=False, allow_none=True) + self.host, self.port = self.socket.getsockname() + self.connection_token = None + #self.register_introspection_functions() + self.commands = BitBakeServerCommands(self) + self.autoregister_all_functions(self.commands, "") + self.interface = interface + + def addcooker(self, cooker): + BaseImplServer.addcooker(self, cooker) + self.commands.cooker = cooker + + def autoregister_all_functions(self, context, prefix): + """ + Convenience method for registering all functions in the scope + of this class that start with a common prefix + """ + methodlist = inspect.getmembers(context, inspect.ismethod) + for name, method in methodlist: + if name.startswith(prefix): + self.register_function(method, name[len(prefix):]) + + + def serve_forever(self): + # Start the actual XMLRPC server + bb.cooker.server_main(self.cooker, self._serve_forever) + + def _serve_forever(self): + """ + Serve Requests. Overloaded to honor a quit command + """ + self.quit = False + while not self.quit: + fds = [self] + nextsleep = 0.1 + for function, data in self._idlefuns.items(): + retval = None + try: + retval = function(self, data, False) + if retval is False: + del self._idlefuns[function] + elif retval is True: + nextsleep = 0 + elif isinstance(retval, float): + if (retval < nextsleep): + nextsleep = retval + else: + fds = fds + retval + except SystemExit: + raise + except: + import traceback + traceback.print_exc() + if retval == None: + # the function execute failed; delete it + del self._idlefuns[function] + pass + + socktimeout = self.socket.gettimeout() or nextsleep + socktimeout = min(socktimeout, nextsleep) + # Mirror what BaseServer handle_request would do + try: + fd_sets = select.select(fds, [], [], socktimeout) + if fd_sets[0] and self in fd_sets[0]: + self._handle_request_noblock() + except IOError: + # we ignore interrupted calls + pass + + # Tell idle functions we're exiting + for function, data in self._idlefuns.items(): + try: + retval = function(self, data, True) + except: + pass + self.server_close() + return + + def set_connection_token(self, token): + self.connection_token = token + +class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection): + def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = None): + self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port) + self.clientinfo = clientinfo + self.serverImpl = serverImpl + self.observer_only = observer_only + if featureset: + self.featureset = featureset + else: + self.featureset = [] + + def connect(self, token = None): + if token is None: + if self.observer_only: + token = "observer" + else: + token = self.connection.addClient() + + if token is None: + return None + + self.transport.set_connection_token(token) + return self + + def setupEventQueue(self): + self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo) + for event in bb.event.ui_queue: + self.events.queue_event(event) + + _, error = self.connection.runCommand(["setFeatures", self.featureset]) + if error: + # disconnect the client, we can't make the setFeature work + self.connection.removeClient() + # no need to log it here, the error shall be sent to the client + raise BaseException(error) + + def removeClient(self): + if not self.observer_only: + self.connection.removeClient() + + def terminate(self): + # Don't wait for server indefinitely + import socket + socket.setdefaulttimeout(2) + try: + self.events.system_quit() + except: + pass + try: + self.connection.removeClient() + except: + pass + +class BitBakeServer(BitBakeBaseServer): + def initServer(self, interface = ("localhost", 0), single_use = False): + self.interface = interface + self.serverImpl = XMLRPCServer(interface, single_use) + + def detach(self): + daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log") + del self.cooker + + def establishConnection(self, featureset): + self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset) + return self.connection.connect() + + def set_connection_token(self, token): + self.connection.transport.set_connection_token(token) + +class BitBakeXMLRPCClient(BitBakeBaseServer): + + def __init__(self, observer_only = False, token = None): + self.token = token + + self.observer_only = observer_only + # if we need extra caches, just tell the server to load them all + pass + + def saveConnectionDetails(self, remote): + self.remote = remote + + def establishConnection(self, featureset): + # The format of "remote" must be "server:port" + try: + [host, port] = self.remote.split(":") + port = int(port) + except Exception as e: + bb.warn("Failed to read remote definition (%s)" % str(e)) + raise e + + # We need our IP for the server connection. We get the IP + # by trying to connect with the server + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect((host, port)) + ip = s.getsockname()[0] + s.close() + except Exception as e: + bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e))) + raise e + try: + self.serverImpl = XMLRPCProxyServer(host, port) + self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset) + return self.connection.connect(self.token) + except Exception as e: + bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e))) + raise e + + def endSession(self): + self.connection.removeClient() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/shell.py b/import-layers/yocto-poky/bitbake/lib/bb/shell.py new file mode 100644 index 000000000..1dd8d54bd --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/shell.py @@ -0,0 +1,820 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +########################################################################## +# +# Copyright (C) 2005-2006 Michael 'Mickey' Lauer <mickey@Vanille.de> +# Copyright (C) 2005-2006 Vanille Media +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +########################################################################## +# +# Thanks to: +# * Holger Freyther <zecke@handhelds.org> +# * Justin Patrin <papercrane@reversefold.com> +# +########################################################################## + +""" +BitBake Shell + +IDEAS: + * list defined tasks per package + * list classes + * toggle force + * command to reparse just one (or more) bbfile(s) + * automatic check if reparsing is necessary (inotify?) + * frontend for bb file manipulation + * more shell-like features: + - output control, i.e. pipe output into grep, sort, etc. + - job control, i.e. bring running commands into background and foreground + * start parsing in background right after startup + * ncurses interface + +PROBLEMS: + * force doesn't always work + * readline completion for commands with more than one parameters + +""" + +########################################################################## +# Import and setup global variables +########################################################################## + +from __future__ import print_function +from functools import reduce +try: + set +except NameError: + from sets import Set as set +import sys, os, readline, socket, httplib, urllib, commands, popen2, shlex, Queue, fnmatch +from bb import data, parse, build, cache, taskdata, runqueue, providers as Providers + +__version__ = "0.5.3.1" +__credits__ = """BitBake Shell Version %s (C) 2005 Michael 'Mickey' Lauer <mickey@Vanille.de> +Type 'help' for more information, press CTRL-D to exit.""" % __version__ + +cmds = {} +leave_mainloop = False +last_exception = None +cooker = None +parsed = False +debug = os.environ.get( "BBSHELL_DEBUG", "" ) + +########################################################################## +# Class BitBakeShellCommands +########################################################################## + +class BitBakeShellCommands: + """This class contains the valid commands for the shell""" + + def __init__( self, shell ): + """Register all the commands""" + self._shell = shell + for attr in BitBakeShellCommands.__dict__: + if not attr.startswith( "_" ): + if attr.endswith( "_" ): + command = attr[:-1].lower() + else: + command = attr[:].lower() + method = getattr( BitBakeShellCommands, attr ) + debugOut( "registering command '%s'" % command ) + # scan number of arguments + usage = getattr( method, "usage", "" ) + if usage != "<...>": + numArgs = len( usage.split() ) + else: + numArgs = -1 + shell.registerCommand( command, method, numArgs, "%s %s" % ( command, usage ), method.__doc__ ) + + def _checkParsed( self ): + if not parsed: + print("SHELL: This command needs to parse bbfiles...") + self.parse( None ) + + def _findProvider( self, item ): + self._checkParsed() + # Need to use taskData for this information + preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 ) + if not preferred: preferred = item + try: + lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status) + except KeyError: + if item in cooker.status.providers: + pf = cooker.status.providers[item][0] + else: + pf = None + return pf + + def alias( self, params ): + """Register a new name for a command""" + new, old = params + if not old in cmds: + print("ERROR: Command '%s' not known" % old) + else: + cmds[new] = cmds[old] + print("OK") + alias.usage = "<alias> <command>" + + def buffer( self, params ): + """Dump specified output buffer""" + index = params[0] + print(self._shell.myout.buffer( int( index ) )) + buffer.usage = "<index>" + + def buffers( self, params ): + """Show the available output buffers""" + commands = self._shell.myout.bufferedCommands() + if not commands: + print("SHELL: No buffered commands available yet. Start doing something.") + else: + print("="*35, "Available Output Buffers", "="*27) + for index, cmd in enumerate( commands ): + print("| %s %s" % ( str( index ).ljust( 3 ), cmd )) + print("="*88) + + def build( self, params, cmd = "build" ): + """Build a providee""" + global last_exception + globexpr = params[0] + self._checkParsed() + names = globfilter( cooker.status.pkg_pn, globexpr ) + if len( names ) == 0: names = [ globexpr ] + print("SHELL: Building %s" % ' '.join( names )) + + td = taskdata.TaskData(cooker.configuration.abort) + localdata = data.createCopy(cooker.configuration.data) + data.update_data(localdata) + data.expandKeys(localdata) + + try: + tasks = [] + for name in names: + td.add_provider(localdata, cooker.status, name) + providers = td.get_provider(name) + + if len(providers) == 0: + raise Providers.NoProvider + + tasks.append([name, "do_%s" % cmd]) + + td.add_unresolved(localdata, cooker.status) + + rq = runqueue.RunQueue(cooker, localdata, cooker.status, td, tasks) + rq.prepare_runqueue() + rq.execute_runqueue() + + except Providers.NoProvider: + print("ERROR: No Provider") + last_exception = Providers.NoProvider + + except runqueue.TaskFailure as fnids: + last_exception = runqueue.TaskFailure + + except build.FuncFailed as e: + print("ERROR: Couldn't build '%s'" % names) + last_exception = e + + + build.usage = "<providee>" + + def clean( self, params ): + """Clean a providee""" + self.build( params, "clean" ) + clean.usage = "<providee>" + + def compile( self, params ): + """Execute 'compile' on a providee""" + self.build( params, "compile" ) + compile.usage = "<providee>" + + def configure( self, params ): + """Execute 'configure' on a providee""" + self.build( params, "configure" ) + configure.usage = "<providee>" + + def install( self, params ): + """Execute 'install' on a providee""" + self.build( params, "install" ) + install.usage = "<providee>" + + def edit( self, params ): + """Call $EDITOR on a providee""" + name = params[0] + bbfile = self._findProvider( name ) + if bbfile is not None: + os.system( "%s %s" % ( os.environ.get( "EDITOR", "vi" ), bbfile ) ) + else: + print("ERROR: Nothing provides '%s'" % name) + edit.usage = "<providee>" + + def environment( self, params ): + """Dump out the outer BitBake environment""" + cooker.showEnvironment() + + def exit_( self, params ): + """Leave the BitBake Shell""" + debugOut( "setting leave_mainloop to true" ) + global leave_mainloop + leave_mainloop = True + + def fetch( self, params ): + """Fetch a providee""" + self.build( params, "fetch" ) + fetch.usage = "<providee>" + + def fileBuild( self, params, cmd = "build" ): + """Parse and build a .bb file""" + global last_exception + name = params[0] + bf = completeFilePath( name ) + print("SHELL: Calling '%s' on '%s'" % ( cmd, bf )) + + try: + cooker.buildFile(bf, cmd) + except parse.ParseError: + print("ERROR: Unable to open or parse '%s'" % bf) + except build.FuncFailed as e: + print("ERROR: Couldn't build '%s'" % name) + last_exception = e + + fileBuild.usage = "<bbfile>" + + def fileClean( self, params ): + """Clean a .bb file""" + self.fileBuild( params, "clean" ) + fileClean.usage = "<bbfile>" + + def fileEdit( self, params ): + """Call $EDITOR on a .bb file""" + name = params[0] + os.system( "%s %s" % ( os.environ.get( "EDITOR", "vi" ), completeFilePath( name ) ) ) + fileEdit.usage = "<bbfile>" + + def fileRebuild( self, params ): + """Rebuild (clean & build) a .bb file""" + self.fileBuild( params, "rebuild" ) + fileRebuild.usage = "<bbfile>" + + def fileReparse( self, params ): + """(re)Parse a bb file""" + bbfile = params[0] + print("SHELL: Parsing '%s'" % bbfile) + parse.update_mtime( bbfile ) + cooker.parser.reparse(bbfile) + if False: #fromCache: + print("SHELL: File has not been updated, not reparsing") + else: + print("SHELL: Parsed") + fileReparse.usage = "<bbfile>" + + def abort( self, params ): + """Toggle abort task execution flag (see bitbake -k)""" + cooker.configuration.abort = not cooker.configuration.abort + print("SHELL: Abort Flag is now '%s'" % repr( cooker.configuration.abort )) + + def force( self, params ): + """Toggle force task execution flag (see bitbake -f)""" + cooker.configuration.force = not cooker.configuration.force + print("SHELL: Force Flag is now '%s'" % repr( cooker.configuration.force )) + + def help( self, params ): + """Show a comprehensive list of commands and their purpose""" + print("="*30, "Available Commands", "="*30) + for cmd in sorted(cmds): + function, numparams, usage, helptext = cmds[cmd] + print("| %s | %s" % (usage.ljust(30), helptext)) + print("="*78) + + def lastError( self, params ): + """Show the reason or log that was produced by the last BitBake event exception""" + if last_exception is None: + print("SHELL: No Errors yet (Phew)...") + else: + reason, event = last_exception.args + print("SHELL: Reason for the last error: '%s'" % reason) + if ':' in reason: + msg, filename = reason.split( ':' ) + filename = filename.strip() + print("SHELL: Dumping log file for last error:") + try: + print(open( filename ).read()) + except IOError: + print("ERROR: Couldn't open '%s'" % filename) + + def match( self, params ): + """Dump all files or providers matching a glob expression""" + what, globexpr = params + if what == "files": + self._checkParsed() + for key in globfilter( cooker.status.pkg_fn, globexpr ): print(key) + elif what == "providers": + self._checkParsed() + for key in globfilter( cooker.status.pkg_pn, globexpr ): print(key) + else: + print("Usage: match %s" % self.print_.usage) + match.usage = "<files|providers> <glob>" + + def new( self, params ): + """Create a new .bb file and open the editor""" + dirname, filename = params + packages = '/'.join( data.getVar( "BBFILES", cooker.configuration.data, 1 ).split('/')[:-2] ) + fulldirname = "%s/%s" % ( packages, dirname ) + + if not os.path.exists( fulldirname ): + print("SHELL: Creating '%s'" % fulldirname) + os.mkdir( fulldirname ) + if os.path.exists( fulldirname ) and os.path.isdir( fulldirname ): + if os.path.exists( "%s/%s" % ( fulldirname, filename ) ): + print("SHELL: ERROR: %s/%s already exists" % ( fulldirname, filename )) + return False + print("SHELL: Creating '%s/%s'" % ( fulldirname, filename )) + newpackage = open( "%s/%s" % ( fulldirname, filename ), "w" ) + print("""DESCRIPTION = "" +SECTION = "" +AUTHOR = "" +HOMEPAGE = "" +MAINTAINER = "" +LICENSE = "GPL" +PR = "r0" + +SRC_URI = "" + +#inherit base + +#do_configure() { +# +#} + +#do_compile() { +# +#} + +#do_stage() { +# +#} + +#do_install() { +# +#} +""", file=newpackage) + newpackage.close() + os.system( "%s %s/%s" % ( os.environ.get( "EDITOR" ), fulldirname, filename ) ) + new.usage = "<directory> <filename>" + + def package( self, params ): + """Execute 'package' on a providee""" + self.build( params, "package" ) + package.usage = "<providee>" + + def pasteBin( self, params ): + """Send a command + output buffer to the pastebin at http://rafb.net/paste""" + index = params[0] + contents = self._shell.myout.buffer( int( index ) ) + sendToPastebin( "output of " + params[0], contents ) + pasteBin.usage = "<index>" + + def pasteLog( self, params ): + """Send the last event exception error log (if there is one) to http://rafb.net/paste""" + if last_exception is None: + print("SHELL: No Errors yet (Phew)...") + else: + reason, event = last_exception.args + print("SHELL: Reason for the last error: '%s'" % reason) + if ':' in reason: + msg, filename = reason.split( ':' ) + filename = filename.strip() + print("SHELL: Pasting log file to pastebin...") + + file = open( filename ).read() + sendToPastebin( "contents of " + filename, file ) + + def patch( self, params ): + """Execute 'patch' command on a providee""" + self.build( params, "patch" ) + patch.usage = "<providee>" + + def parse( self, params ): + """(Re-)parse .bb files and calculate the dependency graph""" + cooker.status = cache.CacheData(cooker.caches_array) + ignore = data.getVar("ASSUME_PROVIDED", cooker.configuration.data, 1) or "" + cooker.status.ignored_dependencies = set( ignore.split() ) + cooker.handleCollections( data.getVar("BBFILE_COLLECTIONS", cooker.configuration.data, 1) ) + + (filelist, masked) = cooker.collect_bbfiles() + cooker.parse_bbfiles(filelist, masked, cooker.myProgressCallback) + cooker.buildDepgraph() + global parsed + parsed = True + print() + + def reparse( self, params ): + """(re)Parse a providee's bb file""" + bbfile = self._findProvider( params[0] ) + if bbfile is not None: + print("SHELL: Found bbfile '%s' for '%s'" % ( bbfile, params[0] )) + self.fileReparse( [ bbfile ] ) + else: + print("ERROR: Nothing provides '%s'" % params[0]) + reparse.usage = "<providee>" + + def getvar( self, params ): + """Dump the contents of an outer BitBake environment variable""" + var = params[0] + value = data.getVar( var, cooker.configuration.data, 1 ) + print(value) + getvar.usage = "<variable>" + + def peek( self, params ): + """Dump contents of variable defined in providee's metadata""" + name, var = params + bbfile = self._findProvider( name ) + if bbfile is not None: + the_data = cache.Cache.loadDataFull(bbfile, cooker.configuration.data) + value = the_data.getVar( var, 1 ) + print(value) + else: + print("ERROR: Nothing provides '%s'" % name) + peek.usage = "<providee> <variable>" + + def poke( self, params ): + """Set contents of variable defined in providee's metadata""" + name, var, value = params + bbfile = self._findProvider( name ) + if bbfile is not None: + print("ERROR: Sorry, this functionality is currently broken") + #d = cooker.pkgdata[bbfile] + #data.setVar( var, value, d ) + + # mark the change semi persistant + #cooker.pkgdata.setDirty(bbfile, d) + #print "OK" + else: + print("ERROR: Nothing provides '%s'" % name) + poke.usage = "<providee> <variable> <value>" + + def print_( self, params ): + """Dump all files or providers""" + what = params[0] + if what == "files": + self._checkParsed() + for key in cooker.status.pkg_fn: print(key) + elif what == "providers": + self._checkParsed() + for key in cooker.status.providers: print(key) + else: + print("Usage: print %s" % self.print_.usage) + print_.usage = "<files|providers>" + + def python( self, params ): + """Enter the expert mode - an interactive BitBake Python Interpreter""" + sys.ps1 = "EXPERT BB>>> " + sys.ps2 = "EXPERT BB... " + import code + interpreter = code.InteractiveConsole( dict( globals() ) ) + interpreter.interact( "SHELL: Expert Mode - BitBake Python %s\nType 'help' for more information, press CTRL-D to switch back to BBSHELL." % sys.version ) + + def showdata( self, params ): + """Execute 'showdata' on a providee""" + cooker.showEnvironment(None, params) + showdata.usage = "<providee>" + + def setVar( self, params ): + """Set an outer BitBake environment variable""" + var, value = params + data.setVar( var, value, cooker.configuration.data ) + print("OK") + setVar.usage = "<variable> <value>" + + def rebuild( self, params ): + """Clean and rebuild a .bb file or a providee""" + self.build( params, "clean" ) + self.build( params, "build" ) + rebuild.usage = "<providee>" + + def shell( self, params ): + """Execute a shell command and dump the output""" + if params != "": + print(commands.getoutput( " ".join( params ) )) + shell.usage = "<...>" + + def stage( self, params ): + """Execute 'stage' on a providee""" + self.build( params, "populate_staging" ) + stage.usage = "<providee>" + + def status( self, params ): + """<just for testing>""" + print("-" * 78) + print("building list = '%s'" % cooker.building_list) + print("build path = '%s'" % cooker.build_path) + print("consider_msgs_cache = '%s'" % cooker.consider_msgs_cache) + print("build stats = '%s'" % cooker.stats) + if last_exception is not None: print("last_exception = '%s'" % repr( last_exception.args )) + print("memory output contents = '%s'" % self._shell.myout._buffer) + + def test( self, params ): + """<just for testing>""" + print("testCommand called with '%s'" % params) + + def unpack( self, params ): + """Execute 'unpack' on a providee""" + self.build( params, "unpack" ) + unpack.usage = "<providee>" + + def which( self, params ): + """Computes the providers for a given providee""" + # Need to use taskData for this information + item = params[0] + + self._checkParsed() + + preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 ) + if not preferred: preferred = item + + try: + lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status) + except KeyError: + lv, lf, pv, pf = (None,)*4 + + try: + providers = cooker.status.providers[item] + except KeyError: + print("SHELL: ERROR: Nothing provides", preferred) + else: + for provider in providers: + if provider == pf: provider = " (***) %s" % provider + else: provider = " %s" % provider + print(provider) + which.usage = "<providee>" + +########################################################################## +# Common helper functions +########################################################################## + +def completeFilePath( bbfile ): + """Get the complete bbfile path""" + if not cooker.status: return bbfile + if not cooker.status.pkg_fn: return bbfile + for key in cooker.status.pkg_fn: + if key.endswith( bbfile ): + return key + return bbfile + +def sendToPastebin( desc, content ): + """Send content to http://oe.pastebin.com""" + mydata = {} + mydata["lang"] = "Plain Text" + mydata["desc"] = desc + mydata["cvt_tabs"] = "No" + mydata["nick"] = "%s@%s" % ( os.environ.get( "USER", "unknown" ), socket.gethostname() or "unknown" ) + mydata["text"] = content + params = urllib.urlencode( mydata ) + headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} + + host = "rafb.net" + conn = httplib.HTTPConnection( "%s:80" % host ) + conn.request("POST", "/paste/paste.php", params, headers ) + + response = conn.getresponse() + conn.close() + + if response.status == 302: + location = response.getheader( "location" ) or "unknown" + print("SHELL: Pasted to http://%s%s" % ( host, location )) + else: + print("ERROR: %s %s" % ( response.status, response.reason )) + +def completer( text, state ): + """Return a possible readline completion""" + debugOut( "completer called with text='%s', state='%d'" % ( text, state ) ) + + if state == 0: + line = readline.get_line_buffer() + if " " in line: + line = line.split() + # we are in second (or more) argument + if line[0] in cmds and hasattr( cmds[line[0]][0], "usage" ): # known command and usage + u = getattr( cmds[line[0]][0], "usage" ).split()[0] + if u == "<variable>": + allmatches = cooker.configuration.data.keys() + elif u == "<bbfile>": + if cooker.status.pkg_fn is None: allmatches = [ "(No Matches Available. Parsed yet?)" ] + else: allmatches = [ x.split("/")[-1] for x in cooker.status.pkg_fn ] + elif u == "<providee>": + if cooker.status.pkg_fn is None: allmatches = [ "(No Matches Available. Parsed yet?)" ] + else: allmatches = cooker.status.providers.iterkeys() + else: allmatches = [ "(No tab completion available for this command)" ] + else: allmatches = [ "(No tab completion available for this command)" ] + else: + # we are in first argument + allmatches = cmds.iterkeys() + + completer.matches = [ x for x in allmatches if x[:len(text)] == text ] + #print "completer.matches = '%s'" % completer.matches + if len( completer.matches ) > state: + return completer.matches[state] + else: + return None + +def debugOut( text ): + if debug: + sys.stderr.write( "( %s )\n" % text ) + +def columnize( alist, width = 80 ): + """ + A word-wrap function that preserves existing line breaks + and most spaces in the text. Expects that existing line + breaks are posix newlines (\n). + """ + return reduce(lambda line, word, width=width: '%s%s%s' % + (line, + ' \n'[(len(line[line.rfind('\n')+1:]) + + len(word.split('\n', 1)[0] + ) >= width)], + word), + alist + ) + +def globfilter( names, pattern ): + return fnmatch.filter( names, pattern ) + +########################################################################## +# Class MemoryOutput +########################################################################## + +class MemoryOutput: + """File-like output class buffering the output of the last 10 commands""" + def __init__( self, delegate ): + self.delegate = delegate + self._buffer = [] + self.text = [] + self._command = None + + def startCommand( self, command ): + self._command = command + self.text = [] + def endCommand( self ): + if self._command is not None: + if len( self._buffer ) == 10: del self._buffer[0] + self._buffer.append( ( self._command, self.text ) ) + def removeLast( self ): + if self._buffer: + del self._buffer[ len( self._buffer ) - 1 ] + self.text = [] + self._command = None + def lastBuffer( self ): + if self._buffer: + return self._buffer[ len( self._buffer ) -1 ][1] + def bufferedCommands( self ): + return [ cmd for cmd, output in self._buffer ] + def buffer( self, i ): + if i < len( self._buffer ): + return "BB>> %s\n%s" % ( self._buffer[i][0], "".join( self._buffer[i][1] ) ) + else: return "ERROR: Invalid buffer number. Buffer needs to be in (0, %d)" % ( len( self._buffer ) - 1 ) + def write( self, text ): + if self._command is not None and text != "BB>> ": self.text.append( text ) + if self.delegate is not None: self.delegate.write( text ) + def flush( self ): + return self.delegate.flush() + def fileno( self ): + return self.delegate.fileno() + def isatty( self ): + return self.delegate.isatty() + +########################################################################## +# Class BitBakeShell +########################################################################## + +class BitBakeShell: + + def __init__( self ): + """Register commands and set up readline""" + self.commandQ = Queue.Queue() + self.commands = BitBakeShellCommands( self ) + self.myout = MemoryOutput( sys.stdout ) + self.historyfilename = os.path.expanduser( "~/.bbsh_history" ) + self.startupfilename = os.path.expanduser( "~/.bbsh_startup" ) + + readline.set_completer( completer ) + readline.set_completer_delims( " " ) + readline.parse_and_bind("tab: complete") + + try: + readline.read_history_file( self.historyfilename ) + except IOError: + pass # It doesn't exist yet. + + print(__credits__) + + def cleanup( self ): + """Write readline history and clean up resources""" + debugOut( "writing command history" ) + try: + readline.write_history_file( self.historyfilename ) + except: + print("SHELL: Unable to save command history") + + def registerCommand( self, command, function, numparams = 0, usage = "", helptext = "" ): + """Register a command""" + if usage == "": usage = command + if helptext == "": helptext = function.__doc__ or "<not yet documented>" + cmds[command] = ( function, numparams, usage, helptext ) + + def processCommand( self, command, params ): + """Process a command. Check number of params and print a usage string, if appropriate""" + debugOut( "processing command '%s'..." % command ) + try: + function, numparams, usage, helptext = cmds[command] + except KeyError: + print("SHELL: ERROR: '%s' command is not a valid command." % command) + self.myout.removeLast() + else: + if (numparams != -1) and (not len( params ) == numparams): + print("Usage: '%s'" % usage) + return + + result = function( self.commands, params ) + debugOut( "result was '%s'" % result ) + + def processStartupFile( self ): + """Read and execute all commands found in $HOME/.bbsh_startup""" + if os.path.exists( self.startupfilename ): + startupfile = open( self.startupfilename, "r" ) + for cmdline in startupfile: + debugOut( "processing startup line '%s'" % cmdline ) + if not cmdline: + continue + if "|" in cmdline: + print("ERROR: '|' in startup file is not allowed. Ignoring line") + continue + self.commandQ.put( cmdline.strip() ) + + def main( self ): + """The main command loop""" + while not leave_mainloop: + try: + if self.commandQ.empty(): + sys.stdout = self.myout.delegate + cmdline = raw_input( "BB>> " ) + sys.stdout = self.myout + else: + cmdline = self.commandQ.get() + if cmdline: + allCommands = cmdline.split( ';' ) + for command in allCommands: + pipecmd = None + # + # special case for expert mode + if command == 'python': + sys.stdout = self.myout.delegate + self.processCommand( command, "" ) + sys.stdout = self.myout + else: + self.myout.startCommand( command ) + if '|' in command: # disable output + command, pipecmd = command.split( '|' ) + delegate = self.myout.delegate + self.myout.delegate = None + tokens = shlex.split( command, True ) + self.processCommand( tokens[0], tokens[1:] or "" ) + self.myout.endCommand() + if pipecmd is not None: # restore output + self.myout.delegate = delegate + + pipe = popen2.Popen4( pipecmd ) + pipe.tochild.write( "\n".join( self.myout.lastBuffer() ) ) + pipe.tochild.close() + sys.stdout.write( pipe.fromchild.read() ) + # + except EOFError: + print() + return + except KeyboardInterrupt: + print() + +########################################################################## +# Start function - called from the BitBake command line utility +########################################################################## + +def start( aCooker ): + global cooker + cooker = aCooker + bbshell = BitBakeShell() + bbshell.processStartupFile() + bbshell.main() + bbshell.cleanup() + +if __name__ == "__main__": + print("SHELL: Sorry, this program should only be called by BitBake.") diff --git a/import-layers/yocto-poky/bitbake/lib/bb/siggen.py b/import-layers/yocto-poky/bitbake/lib/bb/siggen.py new file mode 100644 index 000000000..88fc0f1d5 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/siggen.py @@ -0,0 +1,601 @@ +import hashlib +import logging +import os +import re +import tempfile +import bb.data +from bb.checksum import FileChecksumCache + +logger = logging.getLogger('BitBake.SigGen') + +try: + import cPickle as pickle +except ImportError: + import pickle + logger.info('Importing cPickle failed. Falling back to a very slow implementation.') + +def init(d): + siggens = [obj for obj in globals().itervalues() + if type(obj) is type and issubclass(obj, SignatureGenerator)] + + desired = d.getVar("BB_SIGNATURE_HANDLER", True) or "noop" + for sg in siggens: + if desired == sg.name: + return sg(d) + break + else: + logger.error("Invalid signature generator '%s', using default 'noop'\n" + "Available generators: %s", desired, + ', '.join(obj.name for obj in siggens)) + return SignatureGenerator(d) + +class SignatureGenerator(object): + """ + """ + name = "noop" + + def __init__(self, data): + self.taskhash = {} + self.runtaskdeps = {} + self.file_checksum_values = {} + self.taints = {} + + def finalise(self, fn, d, varient): + return + + def get_taskhash(self, fn, task, deps, dataCache): + return "0" + + def writeout_file_checksum_cache(self): + """Write/update the file checksum cache onto disk""" + return + + def stampfile(self, stampbase, file_name, taskname, extrainfo): + return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.') + + def stampcleanmask(self, stampbase, file_name, taskname, extrainfo): + return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.') + + def dump_sigtask(self, fn, task, stampbase, runtime): + return + + def invalidate_task(self, task, d, fn): + bb.build.del_stamp(task, d, fn) + + def dump_sigs(self, dataCache, options): + return + + def get_taskdata(self): + return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints) + + def set_taskdata(self, data): + self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints = data + + +class SignatureGeneratorBasic(SignatureGenerator): + """ + """ + name = "basic" + + def __init__(self, data): + self.basehash = {} + self.taskhash = {} + self.taskdeps = {} + self.runtaskdeps = {} + self.file_checksum_values = {} + self.taints = {} + self.gendeps = {} + self.lookupcache = {} + self.pkgnameextract = re.compile("(?P<fn>.*)\..*") + self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split()) + self.taskwhitelist = None + self.init_rundepcheck(data) + checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE", True) + if checksum_cache_file: + self.checksum_cache = FileChecksumCache() + self.checksum_cache.init_cache(data, checksum_cache_file) + else: + self.checksum_cache = None + + def init_rundepcheck(self, data): + self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None + if self.taskwhitelist: + self.twl = re.compile(self.taskwhitelist) + else: + self.twl = None + + def _build_data(self, fn, d): + + tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) + + taskdeps = {} + basehash = {} + + for task in tasklist: + data = lookupcache[task] + + if data is None: + bb.error("Task %s from %s seems to be empty?!" % (task, fn)) + data = '' + + gendeps[task] -= self.basewhitelist + newdeps = gendeps[task] + seen = set() + while newdeps: + nextdeps = newdeps + seen |= nextdeps + newdeps = set() + for dep in nextdeps: + if dep in self.basewhitelist: + continue + gendeps[dep] -= self.basewhitelist + newdeps |= gendeps[dep] + newdeps -= seen + + alldeps = sorted(seen) + for dep in alldeps: + data = data + dep + var = lookupcache[dep] + if var is not None: + data = data + str(var) + self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest() + taskdeps[task] = alldeps + + self.taskdeps[fn] = taskdeps + self.gendeps[fn] = gendeps + self.lookupcache[fn] = lookupcache + + return taskdeps + + def finalise(self, fn, d, variant): + + if variant: + fn = "virtual:" + variant + ":" + fn + + try: + taskdeps = self._build_data(fn, d) + except: + bb.warn("Error during finalise of %s" % fn) + raise + + #Slow but can be useful for debugging mismatched basehashes + #for task in self.taskdeps[fn]: + # self.dump_sigtask(fn, task, d.getVar("STAMP", True), False) + + for task in taskdeps: + d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) + + def rundep_check(self, fn, recipename, task, dep, depname, dataCache): + # Return True if we should keep the dependency, False to drop it + # We only manipulate the dependencies for packages not in the whitelist + if self.twl and not self.twl.search(recipename): + # then process the actual dependencies + if self.twl.search(depname): + return False + return True + + def read_taint(self, fn, task, stampbase): + taint = None + try: + with open(stampbase + '.' + task + '.taint', 'r') as taintf: + taint = taintf.read() + except IOError: + pass + return taint + + def get_taskhash(self, fn, task, deps, dataCache): + k = fn + "." + task + data = dataCache.basetaskhash[k] + self.runtaskdeps[k] = [] + self.file_checksum_values[k] = [] + recipename = dataCache.pkg_fn[fn] + + for dep in sorted(deps, key=clean_basepath): + depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')] + if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): + continue + if dep not in self.taskhash: + bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep) + data = data + self.taskhash[dep] + self.runtaskdeps[k].append(dep) + + if task in dataCache.file_checksums[fn]: + if self.checksum_cache: + checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename) + else: + checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) + for (f,cs) in checksums: + self.file_checksum_values[k].append((f,cs)) + if cs: + data = data + cs + + taskdep = dataCache.task_deps[fn] + if 'nostamp' in taskdep and task in taskdep['nostamp']: + # Nostamp tasks need an implicit taint so that they force any dependent tasks to run + import uuid + taint = str(uuid.uuid4()) + data = data + taint + self.taints[k] = "nostamp:" + taint + + taint = self.read_taint(fn, task, dataCache.stamp[fn]) + if taint: + data = data + taint + self.taints[k] = taint + logger.warn("%s is tainted from a forced run" % k) + + h = hashlib.md5(data).hexdigest() + self.taskhash[k] = h + #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) + return h + + def writeout_file_checksum_cache(self): + """Write/update the file checksum cache onto disk""" + if self.checksum_cache: + self.checksum_cache.save_extras() + self.checksum_cache.save_merge() + else: + bb.fetch2.fetcher_parse_save() + bb.fetch2.fetcher_parse_done() + + def dump_sigtask(self, fn, task, stampbase, runtime): + + k = fn + "." + task + referencestamp = stampbase + if isinstance(runtime, str) and runtime.startswith("customfile"): + sigfile = stampbase + referencestamp = runtime[11:] + elif runtime and k in self.taskhash: + sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k] + else: + sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k] + + bb.utils.mkdirhier(os.path.dirname(sigfile)) + + data = {} + data['task'] = task + data['basewhitelist'] = self.basewhitelist + data['taskwhitelist'] = self.taskwhitelist + data['taskdeps'] = self.taskdeps[fn][task] + data['basehash'] = self.basehash[k] + data['gendeps'] = {} + data['varvals'] = {} + data['varvals'][task] = self.lookupcache[fn][task] + for dep in self.taskdeps[fn][task]: + if dep in self.basewhitelist: + continue + data['gendeps'][dep] = self.gendeps[fn][dep] + data['varvals'][dep] = self.lookupcache[fn][dep] + + if runtime and k in self.taskhash: + data['runtaskdeps'] = self.runtaskdeps[k] + data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k]] + data['runtaskhashes'] = {} + for dep in data['runtaskdeps']: + data['runtaskhashes'][dep] = self.taskhash[dep] + data['taskhash'] = self.taskhash[k] + + taint = self.read_taint(fn, task, referencestamp) + if taint: + data['taint'] = taint + + if runtime and k in self.taints: + if 'nostamp:' in self.taints[k]: + data['taint'] = self.taints[k] + + fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") + try: + with os.fdopen(fd, "wb") as stream: + p = pickle.dump(data, stream, -1) + stream.flush() + os.chmod(tmpfile, 0664) + os.rename(tmpfile, sigfile) + except (OSError, IOError) as err: + try: + os.unlink(tmpfile) + except OSError: + pass + raise err + + computed_basehash = calc_basehash(data) + if computed_basehash != self.basehash[k]: + bb.error("Basehash mismatch %s verses %s for %s" % (computed_basehash, self.basehash[k], k)) + if k in self.taskhash: + computed_taskhash = calc_taskhash(data) + if computed_taskhash != self.taskhash[k]: + bb.error("Taskhash mismatch %s verses %s for %s" % (computed_taskhash, self.taskhash[k], k)) + + + def dump_sigs(self, dataCache, options): + for fn in self.taskdeps: + for task in self.taskdeps[fn]: + k = fn + "." + task + if k not in self.taskhash: + continue + if dataCache.basetaskhash[k] != self.basehash[k]: + bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k) + bb.error("The mismatched hashes were %s and %s" % (dataCache.basetaskhash[k], self.basehash[k])) + self.dump_sigtask(fn, task, dataCache.stamp[fn], True) + +class SignatureGeneratorBasicHash(SignatureGeneratorBasic): + name = "basichash" + + def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False): + if taskname != "do_setscene" and taskname.endswith("_setscene"): + k = fn + "." + taskname[:-9] + else: + k = fn + "." + taskname + if clean: + h = "*" + elif k in self.taskhash: + h = self.taskhash[k] + else: + # If k is not in basehash, then error + h = self.basehash[k] + return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.') + + def stampcleanmask(self, stampbase, fn, taskname, extrainfo): + return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True) + + def invalidate_task(self, task, d, fn): + bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task)) + bb.build.write_taint(task, d, fn) + +def dump_this_task(outfile, d): + import bb.parse + fn = d.getVar("BB_FILENAME", True) + task = "do_" + d.getVar("BB_CURRENTTASK", True) + referencestamp = bb.build.stamp_internal(task, d, None, True) + bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp) + +def clean_basepath(a): + b = a.rsplit("/", 2)[1] + a.rsplit("/", 2)[2] + if a.startswith("virtual:"): + b = b + ":" + a.rsplit(":", 1)[0] + return b + +def clean_basepaths(a): + b = {} + for x in a: + b[clean_basepath(x)] = a[x] + return b + +def clean_basepaths_list(a): + b = [] + for x in a: + b.append(clean_basepath(x)) + return b + +def compare_sigfiles(a, b, recursecb = None): + output = [] + + p1 = pickle.Unpickler(open(a, "rb")) + a_data = p1.load() + p2 = pickle.Unpickler(open(b, "rb")) + b_data = p2.load() + + def dict_diff(a, b, whitelist=set()): + sa = set(a.keys()) + sb = set(b.keys()) + common = sa & sb + changed = set() + for i in common: + if a[i] != b[i] and i not in whitelist: + changed.add(i) + added = sb - sa + removed = sa - sb + return changed, added, removed + + def file_checksums_diff(a, b): + from collections import Counter + # Handle old siginfo format + if isinstance(a, dict): + a = [(os.path.basename(f), cs) for f, cs in a.items()] + if isinstance(b, dict): + b = [(os.path.basename(f), cs) for f, cs in b.items()] + # Compare lists, ensuring we can handle duplicate filenames if they exist + removedcount = Counter(a) + removedcount.subtract(b) + addedcount = Counter(b) + addedcount.subtract(a) + added = [] + for x in b: + if addedcount[x] > 0: + addedcount[x] -= 1 + added.append(x) + removed = [] + changed = [] + for x in a: + if removedcount[x] > 0: + removedcount[x] -= 1 + for y in added: + if y[0] == x[0]: + changed.append((x[0], x[1], y[1])) + added.remove(y) + break + else: + removed.append(x) + added = [x[0] for x in added] + removed = [x[0] for x in removed] + return changed, added, removed + + if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: + output.append("basewhitelist changed from '%s' to '%s'" % (a_data['basewhitelist'], b_data['basewhitelist'])) + if a_data['basewhitelist'] and b_data['basewhitelist']: + output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])) + + if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: + output.append("taskwhitelist changed from '%s' to '%s'" % (a_data['taskwhitelist'], b_data['taskwhitelist'])) + if a_data['taskwhitelist'] and b_data['taskwhitelist']: + output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])) + + if a_data['taskdeps'] != b_data['taskdeps']: + output.append("Task dependencies changed from:\n%s\nto:\n%s" % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) + + if a_data['basehash'] != b_data['basehash']: + output.append("basehash changed from %s to %s" % (a_data['basehash'], b_data['basehash'])) + + changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist']) + if changed: + for dep in changed: + output.append("List of dependencies for variable %s changed from '%s' to '%s'" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) + if a_data['gendeps'][dep] and b_data['gendeps'][dep]: + output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep])) + if added: + for dep in added: + output.append("Dependency on variable %s was added" % (dep)) + if removed: + for dep in removed: + output.append("Dependency on Variable %s was removed" % (dep)) + + + changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals']) + if changed: + for dep in changed: + output.append("Variable %s value changed from '%s' to '%s'" % (dep, a_data['varvals'][dep], b_data['varvals'][dep])) + + changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values']) + if changed: + for f, old, new in changed: + output.append("Checksum for file %s changed from %s to %s" % (f, old, new)) + if added: + for f in added: + output.append("Dependency on checksum of file %s was added" % (f)) + if removed: + for f in removed: + output.append("Dependency on checksum of file %s was removed" % (f)) + + + if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']): + changed = ["Number of task dependencies changed"] + else: + changed = [] + for idx, task in enumerate(a_data['runtaskdeps']): + a = a_data['runtaskdeps'][idx] + b = b_data['runtaskdeps'][idx] + if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b]: + changed.append("%s with hash %s\n changed to\n%s with hash %s" % (a, a_data['runtaskhashes'][a], b, b_data['runtaskhashes'][b])) + + if changed: + output.append("runtaskdeps changed from %s to %s" % (clean_basepaths_list(a_data['runtaskdeps']), clean_basepaths_list(b_data['runtaskdeps']))) + output.append("\n".join(changed)) + + + if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data: + a = a_data['runtaskhashes'] + b = b_data['runtaskhashes'] + changed, added, removed = dict_diff(a, b) + if added: + for dep in added: + bdep_found = False + if removed: + for bdep in removed: + if b[dep] == a[bdep]: + #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep)) + bdep_found = True + if not bdep_found: + output.append("Dependency on task %s was added with hash %s" % (clean_basepath(dep), b[dep])) + if removed: + for dep in removed: + adep_found = False + if added: + for adep in added: + if b[adep] == a[dep]: + #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep)) + adep_found = True + if not adep_found: + output.append("Dependency on task %s was removed with hash %s" % (clean_basepath(dep), a[dep])) + if changed: + for dep in changed: + output.append("Hash for dependent task %s changed from %s to %s" % (clean_basepath(dep), a[dep], b[dep])) + if callable(recursecb): + # If a dependent hash changed, might as well print the line above and then defer to the changes in + # that hash since in all likelyhood, they're the same changes this task also saw. + recout = recursecb(dep, a[dep], b[dep]) + if recout: + output = [output[-1]] + recout + + a_taint = a_data.get('taint', None) + b_taint = b_data.get('taint', None) + if a_taint != b_taint: + output.append("Taint (by forced/invalidated task) changed from %s to %s" % (a_taint, b_taint)) + + return output + + +def calc_basehash(sigdata): + task = sigdata['task'] + basedata = sigdata['varvals'][task] + + if basedata is None: + basedata = '' + + alldeps = sigdata['taskdeps'] + for dep in alldeps: + basedata = basedata + dep + val = sigdata['varvals'][dep] + if val is not None: + basedata = basedata + str(val) + + return hashlib.md5(basedata).hexdigest() + +def calc_taskhash(sigdata): + data = sigdata['basehash'] + + for dep in sigdata['runtaskdeps']: + data = data + sigdata['runtaskhashes'][dep] + + for c in sigdata['file_checksum_values']: + data = data + c[1] + + if 'taint' in sigdata: + if 'nostamp:' in sigdata['taint']: + data = data + sigdata['taint'][8:] + else: + data = data + sigdata['taint'] + + return hashlib.md5(data).hexdigest() + + +def dump_sigfile(a): + output = [] + + p1 = pickle.Unpickler(open(a, "rb")) + a_data = p1.load() + + output.append("basewhitelist: %s" % (a_data['basewhitelist'])) + + output.append("taskwhitelist: %s" % (a_data['taskwhitelist'])) + + output.append("Task dependencies: %s" % (sorted(a_data['taskdeps']))) + + output.append("basehash: %s" % (a_data['basehash'])) + + for dep in a_data['gendeps']: + output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep])) + + for dep in a_data['varvals']: + output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep])) + + if 'runtaskdeps' in a_data: + output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps'])) + + if 'file_checksum_values' in a_data: + output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values'])) + + if 'runtaskhashes' in a_data: + for dep in a_data['runtaskhashes']: + output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep])) + + if 'taint' in a_data: + output.append("Tainted (by forced/invalidated task): %s" % a_data['taint']) + + if 'task' in a_data: + computed_basehash = calc_basehash(a_data) + output.append("Computed base hash is %s and from file %s" % (computed_basehash, a_data['basehash'])) + else: + output.append("Unable to compute base hash") + + computed_taskhash = calc_taskhash(a_data) + output.append("Computed task hash is %s" % computed_taskhash) + + return output diff --git a/import-layers/yocto-poky/bitbake/lib/bb/taskdata.py b/import-layers/yocto-poky/bitbake/lib/bb/taskdata.py new file mode 100644 index 000000000..9ae52d77d --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/taskdata.py @@ -0,0 +1,690 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'TaskData' implementation + +Task data collection and handling + +""" + +# Copyright (C) 2006 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import logging +import re +import bb + +logger = logging.getLogger("BitBake.TaskData") + +def re_match_strings(target, strings): + """ + Whether or not the string 'target' matches + any one string of the strings which can be regular expression string + """ + return any(name == target or re.match(name, target) + for name in strings) + +class TaskData: + """ + BitBake Task Data implementation + """ + def __init__(self, abort = True, tryaltconfigs = False, skiplist = None, allowincomplete = False): + self.build_names_index = [] + self.run_names_index = [] + self.fn_index = [] + + self.build_targets = {} + self.run_targets = {} + + self.external_targets = [] + + self.tasks_fnid = [] + self.tasks_name = [] + self.tasks_tdepends = [] + self.tasks_idepends = [] + self.tasks_irdepends = [] + # Cache to speed up task ID lookups + self.tasks_lookup = {} + + self.depids = {} + self.rdepids = {} + + self.consider_msgs_cache = [] + + self.failed_deps = [] + self.failed_rdeps = [] + self.failed_fnids = [] + + self.abort = abort + self.tryaltconfigs = tryaltconfigs + self.allowincomplete = allowincomplete + + self.skiplist = skiplist + + def getbuild_id(self, name): + """ + Return an ID number for the build target name. + If it doesn't exist, create one. + """ + if not name in self.build_names_index: + self.build_names_index.append(name) + return len(self.build_names_index) - 1 + + return self.build_names_index.index(name) + + def getrun_id(self, name): + """ + Return an ID number for the run target name. + If it doesn't exist, create one. + """ + if not name in self.run_names_index: + self.run_names_index.append(name) + return len(self.run_names_index) - 1 + + return self.run_names_index.index(name) + + def getfn_id(self, name): + """ + Return an ID number for the filename. + If it doesn't exist, create one. + """ + if not name in self.fn_index: + self.fn_index.append(name) + return len(self.fn_index) - 1 + + return self.fn_index.index(name) + + def gettask_ids(self, fnid): + """ + Return an array of the ID numbers matching a given fnid. + """ + ids = [] + if fnid in self.tasks_lookup: + for task in self.tasks_lookup[fnid]: + ids.append(self.tasks_lookup[fnid][task]) + return ids + + def gettask_id_fromfnid(self, fnid, task): + """ + Return an ID number for the task matching fnid and task. + """ + if fnid in self.tasks_lookup: + if task in self.tasks_lookup[fnid]: + return self.tasks_lookup[fnid][task] + + return None + + def gettask_id(self, fn, task, create = True): + """ + Return an ID number for the task matching fn and task. + If it doesn't exist, create one by default. + Optionally return None instead. + """ + fnid = self.getfn_id(fn) + + if fnid in self.tasks_lookup: + if task in self.tasks_lookup[fnid]: + return self.tasks_lookup[fnid][task] + + if not create: + return None + + self.tasks_name.append(task) + self.tasks_fnid.append(fnid) + self.tasks_tdepends.append([]) + self.tasks_idepends.append([]) + self.tasks_irdepends.append([]) + + listid = len(self.tasks_name) - 1 + + if fnid not in self.tasks_lookup: + self.tasks_lookup[fnid] = {} + self.tasks_lookup[fnid][task] = listid + + return listid + + def add_tasks(self, fn, dataCache): + """ + Add tasks for a given fn to the database + """ + + task_deps = dataCache.task_deps[fn] + + fnid = self.getfn_id(fn) + + if fnid in self.failed_fnids: + bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...") + + # Check if we've already seen this fn + if fnid in self.tasks_fnid: + return + + self.add_extra_deps(fn, dataCache) + + for task in task_deps['tasks']: + + # Work out task dependencies + parentids = [] + for dep in task_deps['parents'][task]: + if dep not in task_deps['tasks']: + bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep)) + continue + parentid = self.gettask_id(fn, dep) + parentids.append(parentid) + taskid = self.gettask_id(fn, task) + self.tasks_tdepends[taskid].extend(parentids) + + # Touch all intertask dependencies + if 'depends' in task_deps and task in task_deps['depends']: + ids = [] + for dep in task_deps['depends'][task].split(): + if dep: + if ":" not in dep: + bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep)) + ids.append(((self.getbuild_id(dep.split(":")[0])), dep.split(":")[1])) + self.tasks_idepends[taskid].extend(ids) + if 'rdepends' in task_deps and task in task_deps['rdepends']: + ids = [] + for dep in task_deps['rdepends'][task].split(): + if dep: + if ":" not in dep: + bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep)) + ids.append(((self.getrun_id(dep.split(":")[0])), dep.split(":")[1])) + self.tasks_irdepends[taskid].extend(ids) + + + # Work out build dependencies + if not fnid in self.depids: + dependids = {} + for depend in dataCache.deps[fn]: + dependids[self.getbuild_id(depend)] = None + self.depids[fnid] = dependids.keys() + logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn) + + # Work out runtime dependencies + if not fnid in self.rdepids: + rdependids = {} + rdepends = dataCache.rundeps[fn] + rrecs = dataCache.runrecs[fn] + rdependlist = [] + rreclist = [] + for package in rdepends: + for rdepend in rdepends[package]: + rdependlist.append(rdepend) + rdependids[self.getrun_id(rdepend)] = None + for package in rrecs: + for rdepend in rrecs[package]: + rreclist.append(rdepend) + rdependids[self.getrun_id(rdepend)] = None + if rdependlist: + logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn) + if rreclist: + logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn) + self.rdepids[fnid] = rdependids.keys() + + for dep in self.depids[fnid]: + if dep in self.failed_deps: + self.fail_fnid(fnid) + return + for dep in self.rdepids[fnid]: + if dep in self.failed_rdeps: + self.fail_fnid(fnid) + return + + def add_extra_deps(self, fn, dataCache): + func = dataCache.extradepsfunc.get(fn, None) + if func: + bb.providers.buildWorldTargetList(dataCache) + pn = dataCache.pkg_fn[fn] + params = {'deps': dataCache.deps[fn], + 'world_target': dataCache.world_target, + 'pkg_pn': dataCache.pkg_pn, + 'self_pn': pn} + funcname = '_%s_calculate_extra_depends' % pn.replace('-', '_') + paramlist = ','.join(params.keys()) + func = 'def %s(%s):\n%s\n\n%s(%s)' % (funcname, paramlist, func, funcname, paramlist) + bb.utils.better_exec(func, params) + + + def have_build_target(self, target): + """ + Have we a build target matching this name? + """ + targetid = self.getbuild_id(target) + + if targetid in self.build_targets: + return True + return False + + def have_runtime_target(self, target): + """ + Have we a runtime target matching this name? + """ + targetid = self.getrun_id(target) + + if targetid in self.run_targets: + return True + return False + + def add_build_target(self, fn, item): + """ + Add a build target. + If already present, append the provider fn to the list + """ + targetid = self.getbuild_id(item) + fnid = self.getfn_id(fn) + + if targetid in self.build_targets: + if fnid in self.build_targets[targetid]: + return + self.build_targets[targetid].append(fnid) + return + self.build_targets[targetid] = [fnid] + + def add_runtime_target(self, fn, item): + """ + Add a runtime target. + If already present, append the provider fn to the list + """ + targetid = self.getrun_id(item) + fnid = self.getfn_id(fn) + + if targetid in self.run_targets: + if fnid in self.run_targets[targetid]: + return + self.run_targets[targetid].append(fnid) + return + self.run_targets[targetid] = [fnid] + + def mark_external_target(self, item): + """ + Mark a build target as being externally requested + """ + targetid = self.getbuild_id(item) + + if targetid not in self.external_targets: + self.external_targets.append(targetid) + + def get_unresolved_build_targets(self, dataCache): + """ + Return a list of build targets who's providers + are unknown. + """ + unresolved = [] + for target in self.build_names_index: + if re_match_strings(target, dataCache.ignored_dependencies): + continue + if self.build_names_index.index(target) in self.failed_deps: + continue + if not self.have_build_target(target): + unresolved.append(target) + return unresolved + + def get_unresolved_run_targets(self, dataCache): + """ + Return a list of runtime targets who's providers + are unknown. + """ + unresolved = [] + for target in self.run_names_index: + if re_match_strings(target, dataCache.ignored_dependencies): + continue + if self.run_names_index.index(target) in self.failed_rdeps: + continue + if not self.have_runtime_target(target): + unresolved.append(target) + return unresolved + + def get_provider(self, item): + """ + Return a list of providers of item + """ + targetid = self.getbuild_id(item) + + return self.build_targets[targetid] + + def get_dependees(self, itemid): + """ + Return a list of targets which depend on item + """ + dependees = [] + for fnid in self.depids: + if itemid in self.depids[fnid]: + dependees.append(fnid) + return dependees + + def get_dependees_str(self, item): + """ + Return a list of targets which depend on item as a user readable string + """ + itemid = self.getbuild_id(item) + dependees = [] + for fnid in self.depids: + if itemid in self.depids[fnid]: + dependees.append(self.fn_index[fnid]) + return dependees + + def get_rdependees(self, itemid): + """ + Return a list of targets which depend on runtime item + """ + dependees = [] + for fnid in self.rdepids: + if itemid in self.rdepids[fnid]: + dependees.append(fnid) + return dependees + + def get_rdependees_str(self, item): + """ + Return a list of targets which depend on runtime item as a user readable string + """ + itemid = self.getrun_id(item) + dependees = [] + for fnid in self.rdepids: + if itemid in self.rdepids[fnid]: + dependees.append(self.fn_index[fnid]) + return dependees + + def get_reasons(self, item, runtime=False): + """ + Get the reason(s) for an item not being provided, if any + """ + reasons = [] + if self.skiplist: + for fn in self.skiplist: + skipitem = self.skiplist[fn] + if skipitem.pn == item: + reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason)) + elif runtime and item in skipitem.rprovides: + reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason)) + elif not runtime and item in skipitem.provides: + reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason)) + return reasons + + def get_close_matches(self, item, provider_list): + import difflib + if self.skiplist: + skipped = [] + for fn in self.skiplist: + skipped.append(self.skiplist[fn].pn) + full_list = provider_list + skipped + else: + full_list = provider_list + return difflib.get_close_matches(item, full_list, cutoff=0.7) + + def add_provider(self, cfgData, dataCache, item): + try: + self.add_provider_internal(cfgData, dataCache, item) + except bb.providers.NoProvider: + if self.abort: + raise + self.remove_buildtarget(self.getbuild_id(item)) + + self.mark_external_target(item) + + def add_provider_internal(self, cfgData, dataCache, item): + """ + Add the providers of item to the task data + Mark entries were specifically added externally as against dependencies + added internally during dependency resolution + """ + + if re_match_strings(item, dataCache.ignored_dependencies): + return + + if not item in dataCache.providers: + close_matches = self.get_close_matches(item, dataCache.providers.keys()) + # Is it in RuntimeProviders ? + all_p = bb.providers.getRuntimeProviders(dataCache, item) + for fn in all_p: + new = dataCache.pkg_fn[fn] + " RPROVIDES " + item + if new not in close_matches: + close_matches.append(new) + bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData) + raise bb.providers.NoProvider(item) + + if self.have_build_target(item): + return + + all_p = dataCache.providers[item] + + eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache) + eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids] + + if not eligible: + bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData) + raise bb.providers.NoProvider(item) + + if len(eligible) > 1 and foundUnique == False: + if item not in self.consider_msgs_cache: + providers_list = [] + for fn in eligible: + providers_list.append(dataCache.pkg_fn[fn]) + bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData) + self.consider_msgs_cache.append(item) + + for fn in eligible: + fnid = self.getfn_id(fn) + if fnid in self.failed_fnids: + continue + logger.debug(2, "adding %s to satisfy %s", fn, item) + self.add_build_target(fn, item) + self.add_tasks(fn, dataCache) + + + #item = dataCache.pkg_fn[fn] + + def add_rprovider(self, cfgData, dataCache, item): + """ + Add the runtime providers of item to the task data + (takes item names from RDEPENDS/PACKAGES namespace) + """ + + if re_match_strings(item, dataCache.ignored_dependencies): + return + + if self.have_runtime_target(item): + return + + all_p = bb.providers.getRuntimeProviders(dataCache, item) + + if not all_p: + bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=self.get_reasons(item, True)), cfgData) + raise bb.providers.NoRProvider(item) + + eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache) + eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids] + + if not eligible: + bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData) + raise bb.providers.NoRProvider(item) + + if len(eligible) > 1 and numberPreferred == 0: + if item not in self.consider_msgs_cache: + providers_list = [] + for fn in eligible: + providers_list.append(dataCache.pkg_fn[fn]) + bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData) + self.consider_msgs_cache.append(item) + + if numberPreferred > 1: + if item not in self.consider_msgs_cache: + providers_list = [] + for fn in eligible: + providers_list.append(dataCache.pkg_fn[fn]) + bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData) + self.consider_msgs_cache.append(item) + raise bb.providers.MultipleRProvider(item) + + # run through the list until we find one that we can build + for fn in eligible: + fnid = self.getfn_id(fn) + if fnid in self.failed_fnids: + continue + logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item) + self.add_runtime_target(fn, item) + self.add_tasks(fn, dataCache) + + def fail_fnid(self, fnid, missing_list=None): + """ + Mark a file as failed (unbuildable) + Remove any references from build and runtime provider lists + + missing_list, A list of missing requirements for this target + """ + if fnid in self.failed_fnids: + return + if not missing_list: + missing_list = [] + logger.debug(1, "File '%s' is unbuildable, removing...", self.fn_index[fnid]) + self.failed_fnids.append(fnid) + for target in self.build_targets: + if fnid in self.build_targets[target]: + self.build_targets[target].remove(fnid) + if len(self.build_targets[target]) == 0: + self.remove_buildtarget(target, missing_list) + for target in self.run_targets: + if fnid in self.run_targets[target]: + self.run_targets[target].remove(fnid) + if len(self.run_targets[target]) == 0: + self.remove_runtarget(target, missing_list) + + def remove_buildtarget(self, targetid, missing_list=None): + """ + Mark a build target as failed (unbuildable) + Trigger removal of any files that have this as a dependency + """ + if not missing_list: + missing_list = [self.build_names_index[targetid]] + else: + missing_list = [self.build_names_index[targetid]] + missing_list + logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.build_names_index[targetid], missing_list) + self.failed_deps.append(targetid) + dependees = self.get_dependees(targetid) + for fnid in dependees: + self.fail_fnid(fnid, missing_list) + for taskid in xrange(len(self.tasks_idepends)): + idepends = self.tasks_idepends[taskid] + for (idependid, idependtask) in idepends: + if idependid == targetid: + self.fail_fnid(self.tasks_fnid[taskid], missing_list) + + if self.abort and targetid in self.external_targets: + target = self.build_names_index[targetid] + logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list) + raise bb.providers.NoProvider(target) + + def remove_runtarget(self, targetid, missing_list=None): + """ + Mark a run target as failed (unbuildable) + Trigger removal of any files that have this as a dependency + """ + if not missing_list: + missing_list = [self.run_names_index[targetid]] + else: + missing_list = [self.run_names_index[targetid]] + missing_list + + logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.run_names_index[targetid], missing_list) + self.failed_rdeps.append(targetid) + dependees = self.get_rdependees(targetid) + for fnid in dependees: + self.fail_fnid(fnid, missing_list) + for taskid in xrange(len(self.tasks_irdepends)): + irdepends = self.tasks_irdepends[taskid] + for (idependid, idependtask) in irdepends: + if idependid == targetid: + self.fail_fnid(self.tasks_fnid[taskid], missing_list) + + def add_unresolved(self, cfgData, dataCache): + """ + Resolve all unresolved build and runtime targets + """ + logger.info("Resolving any missing task queue dependencies") + while True: + added = 0 + for target in self.get_unresolved_build_targets(dataCache): + try: + self.add_provider_internal(cfgData, dataCache, target) + added = added + 1 + except bb.providers.NoProvider: + targetid = self.getbuild_id(target) + if self.abort and targetid in self.external_targets and not self.allowincomplete: + raise + if not self.allowincomplete: + self.remove_buildtarget(targetid) + for target in self.get_unresolved_run_targets(dataCache): + try: + self.add_rprovider(cfgData, dataCache, target) + added = added + 1 + except (bb.providers.NoRProvider, bb.providers.MultipleRProvider): + self.remove_runtarget(self.getrun_id(target)) + logger.debug(1, "Resolved " + str(added) + " extra dependencies") + if added == 0: + break + # self.dump_data() + + def get_providermap(self, prefix=None): + provmap = {} + for name in self.build_names_index: + if prefix and not name.startswith(prefix): + continue + if self.have_build_target(name): + provider = self.get_provider(name) + if provider: + provmap[name] = self.fn_index[provider[0]] + return provmap + + def dump_data(self): + """ + Dump some debug information on the internal data structures + """ + logger.debug(3, "build_names:") + logger.debug(3, ", ".join(self.build_names_index)) + + logger.debug(3, "run_names:") + logger.debug(3, ", ".join(self.run_names_index)) + + logger.debug(3, "build_targets:") + for buildid in xrange(len(self.build_names_index)): + target = self.build_names_index[buildid] + targets = "None" + if buildid in self.build_targets: + targets = self.build_targets[buildid] + logger.debug(3, " (%s)%s: %s", buildid, target, targets) + + logger.debug(3, "run_targets:") + for runid in xrange(len(self.run_names_index)): + target = self.run_names_index[runid] + targets = "None" + if runid in self.run_targets: + targets = self.run_targets[runid] + logger.debug(3, " (%s)%s: %s", runid, target, targets) + + logger.debug(3, "tasks:") + for task in xrange(len(self.tasks_name)): + logger.debug(3, " (%s)%s - %s: %s", + task, + self.fn_index[self.tasks_fnid[task]], + self.tasks_name[task], + self.tasks_tdepends[task]) + + logger.debug(3, "dependency ids (per fn):") + for fnid in self.depids: + logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.depids[fnid]) + + logger.debug(3, "runtime dependency ids (per fn):") + for fnid in self.rdepids: + logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.rdepids[fnid]) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/codeparser.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/codeparser.py new file mode 100644 index 000000000..bb820e403 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/codeparser.py @@ -0,0 +1,380 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Test for codeparser.py +# +# Copyright (C) 2010 Chris Larson +# Copyright (C) 2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import logging +import bb + +logger = logging.getLogger('BitBake.TestCodeParser') + +# bb.data references bb.parse but can't directly import due to circular dependencies. +# Hack around it for now :( +import bb.parse +import bb.data + +class ReferenceTest(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + + def setEmptyVars(self, varlist): + for k in varlist: + self.d.setVar(k, "") + + def setValues(self, values): + for k, v in values.items(): + self.d.setVar(k, v) + + def assertReferences(self, refs): + self.assertEqual(self.references, refs) + + def assertExecs(self, execs): + self.assertEqual(self.execs, execs) + +class VariableReferenceTest(ReferenceTest): + + def parseExpression(self, exp): + parsedvar = self.d.expandWithRefs(exp, None) + self.references = parsedvar.references + + def test_simple_reference(self): + self.setEmptyVars(["FOO"]) + self.parseExpression("${FOO}") + self.assertReferences(set(["FOO"])) + + def test_nested_reference(self): + self.setEmptyVars(["BAR"]) + self.d.setVar("FOO", "BAR") + self.parseExpression("${${FOO}}") + self.assertReferences(set(["FOO", "BAR"])) + + def test_python_reference(self): + self.setEmptyVars(["BAR"]) + self.parseExpression("${@bb.data.getVar('BAR', d, True) + 'foo'}") + self.assertReferences(set(["BAR"])) + +class ShellReferenceTest(ReferenceTest): + + def parseExpression(self, exp): + parsedvar = self.d.expandWithRefs(exp, None) + parser = bb.codeparser.ShellParser("ParserTest", logger) + parser.parse_shell(parsedvar.value) + + self.references = parsedvar.references + self.execs = parser.execs + + def test_quotes_inside_assign(self): + self.parseExpression('foo=foo"bar"baz') + self.assertReferences(set([])) + + def test_quotes_inside_arg(self): + self.parseExpression('sed s#"bar baz"#"alpha beta"#g') + self.assertExecs(set(["sed"])) + + def test_arg_continuation(self): + self.parseExpression("sed -i -e s,foo,bar,g \\\n *.pc") + self.assertExecs(set(["sed"])) + + def test_dollar_in_quoted(self): + self.parseExpression('sed -i -e "foo$" *.pc') + self.assertExecs(set(["sed"])) + + def test_quotes_inside_arg_continuation(self): + self.setEmptyVars(["bindir", "D", "libdir"]) + self.parseExpression(""" +sed -i -e s#"moc_location=.*$"#"moc_location=${bindir}/moc4"# \\ +-e s#"uic_location=.*$"#"uic_location=${bindir}/uic4"# \\ +${D}${libdir}/pkgconfig/*.pc +""") + self.assertReferences(set(["bindir", "D", "libdir"])) + + def test_assign_subshell_expansion(self): + self.parseExpression("foo=$(echo bar)") + self.assertExecs(set(["echo"])) + + def test_shell_unexpanded(self): + self.setEmptyVars(["QT_BASE_NAME"]) + self.parseExpression('echo "${QT_BASE_NAME}"') + self.assertExecs(set(["echo"])) + self.assertReferences(set(["QT_BASE_NAME"])) + + def test_incomplete_varexp_single_quotes(self): + self.parseExpression("sed -i -e 's:IP{:I${:g' $pc") + self.assertExecs(set(["sed"])) + + + def test_until(self): + self.parseExpression("until false; do echo true; done") + self.assertExecs(set(["false", "echo"])) + self.assertReferences(set()) + + def test_case(self): + self.parseExpression(""" +case $foo in +*) +bar +;; +esac +""") + self.assertExecs(set(["bar"])) + self.assertReferences(set()) + + def test_assign_exec(self): + self.parseExpression("a=b c='foo bar' alpha 1 2 3") + self.assertExecs(set(["alpha"])) + + def test_redirect_to_file(self): + self.setEmptyVars(["foo"]) + self.parseExpression("echo foo >${foo}/bar") + self.assertExecs(set(["echo"])) + self.assertReferences(set(["foo"])) + + def test_heredoc(self): + self.setEmptyVars(["theta"]) + self.parseExpression(""" +cat <<END +alpha +beta +${theta} +END +""") + self.assertReferences(set(["theta"])) + + def test_redirect_from_heredoc(self): + v = ["B", "SHADOW_MAILDIR", "SHADOW_MAILFILE", "SHADOW_UTMPDIR", "SHADOW_LOGDIR", "bindir"] + self.setEmptyVars(v) + self.parseExpression(""" +cat <<END >${B}/cachedpaths +shadow_cv_maildir=${SHADOW_MAILDIR} +shadow_cv_mailfile=${SHADOW_MAILFILE} +shadow_cv_utmpdir=${SHADOW_UTMPDIR} +shadow_cv_logdir=${SHADOW_LOGDIR} +shadow_cv_passwd_dir=${bindir} +END +""") + self.assertReferences(set(v)) + self.assertExecs(set(["cat"])) + +# def test_incomplete_command_expansion(self): +# self.assertRaises(reftracker.ShellSyntaxError, reftracker.execs, +# bbvalue.shparse("cp foo`", self.d), self.d) + +# def test_rogue_dollarsign(self): +# self.setValues({"D" : "/tmp"}) +# self.parseExpression("install -d ${D}$") +# self.assertReferences(set(["D"])) +# self.assertExecs(set(["install"])) + + +class PythonReferenceTest(ReferenceTest): + + def setUp(self): + self.d = bb.data.init() + if hasattr(bb.utils, "_context"): + self.context = bb.utils._context + else: + import __builtin__ + self.context = __builtin__.__dict__ + + def parseExpression(self, exp): + parsedvar = self.d.expandWithRefs(exp, None) + parser = bb.codeparser.PythonParser("ParserTest", logger) + parser.parse_python(parsedvar.value) + + self.references = parsedvar.references | parser.references + self.execs = parser.execs + + @staticmethod + def indent(value): + """Python Snippets have to be indented, python values don't have to +be. These unit tests are testing snippets.""" + return " " + value + + def test_getvar_reference(self): + self.parseExpression("bb.data.getVar('foo', d, True)") + self.assertReferences(set(["foo"])) + self.assertExecs(set()) + + def test_getvar_computed_reference(self): + self.parseExpression("bb.data.getVar('f' + 'o' + 'o', d, True)") + self.assertReferences(set()) + self.assertExecs(set()) + + def test_getvar_exec_reference(self): + self.parseExpression("eval('bb.data.getVar(\"foo\", d, True)')") + self.assertReferences(set()) + self.assertExecs(set(["eval"])) + + def test_var_reference(self): + self.context["foo"] = lambda x: x + self.setEmptyVars(["FOO"]) + self.parseExpression("foo('${FOO}')") + self.assertReferences(set(["FOO"])) + self.assertExecs(set(["foo"])) + del self.context["foo"] + + def test_var_exec(self): + for etype in ("func", "task"): + self.d.setVar("do_something", "echo 'hi mom! ${FOO}'") + self.d.setVarFlag("do_something", etype, True) + self.parseExpression("bb.build.exec_func('do_something', d)") + self.assertReferences(set([])) + self.assertExecs(set(["do_something"])) + + def test_function_reference(self): + self.context["testfunc"] = lambda msg: bb.msg.note(1, None, msg) + self.d.setVar("FOO", "Hello, World!") + self.parseExpression("testfunc('${FOO}')") + self.assertReferences(set(["FOO"])) + self.assertExecs(set(["testfunc"])) + del self.context["testfunc"] + + def test_qualified_function_reference(self): + self.parseExpression("time.time()") + self.assertExecs(set(["time.time"])) + + def test_qualified_function_reference_2(self): + self.parseExpression("os.path.dirname('/foo/bar')") + self.assertExecs(set(["os.path.dirname"])) + + def test_qualified_function_reference_nested(self): + self.parseExpression("time.strftime('%Y%m%d',time.gmtime())") + self.assertExecs(set(["time.strftime", "time.gmtime"])) + + def test_function_reference_chained(self): + self.context["testget"] = lambda: "\tstrip me " + self.parseExpression("testget().strip()") + self.assertExecs(set(["testget"])) + del self.context["testget"] + + +class DependencyReferenceTest(ReferenceTest): + + pydata = """ +bb.data.getVar('somevar', d, True) +def test(d): + foo = 'bar %s' % 'foo' +def test2(d): + d.getVar(foo, True) + d.getVar('bar', False) + test2(d) + +def a(): + \"\"\"some + stuff + \"\"\" + return "heh" + +test(d) + +bb.data.expand(bb.data.getVar("something", False, d), d) +bb.data.expand("${inexpand} somethingelse", d) +bb.data.getVar(a(), d, False) +""" + + def test_python(self): + self.d.setVar("FOO", self.pydata) + self.setEmptyVars(["inexpand", "a", "test2", "test"]) + self.d.setVarFlags("FOO", { + "func": True, + "python": True, + "lineno": 1, + "filename": "example.bb", + }) + + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + + self.assertEquals(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"])) + + + shelldata = """ +foo () { +bar +} +{ +echo baz +$(heh) +eval `moo` +} +a=b +c=d +( +true && false +test -f foo +testval=something +$testval +) || aiee +! inverted +echo ${somevar} + +case foo in +bar) +echo bar +;; +baz) +echo baz +;; +foo*) +echo foo +;; +esac +""" + + def test_shell(self): + execs = ["bar", "echo", "heh", "moo", "true", "aiee"] + self.d.setVar("somevar", "heh") + self.d.setVar("inverted", "echo inverted...") + self.d.setVarFlag("inverted", "func", True) + self.d.setVar("FOO", self.shelldata) + self.d.setVarFlags("FOO", {"func": True}) + self.setEmptyVars(execs) + + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + + self.assertEquals(deps, set(["somevar", "inverted"] + execs)) + + + def test_vardeps(self): + self.d.setVar("oe_libinstall", "echo test") + self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") + self.d.setVarFlag("FOO", "vardeps", "oe_libinstall") + + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + + self.assertEquals(deps, set(["oe_libinstall"])) + + def test_vardeps_expand(self): + self.d.setVar("oe_libinstall", "echo test") + self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") + self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}") + + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + + self.assertEquals(deps, set(["oe_libinstall"])) + + #Currently no wildcard support + #def test_vardeps_wildcards(self): + # self.d.setVar("oe_libinstall", "echo test") + # self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") + # self.d.setVarFlag("FOO", "vardeps", "oe_*") + # self.assertEquals(deps, set(["oe_libinstall"])) + + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/cow.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/cow.py new file mode 100644 index 000000000..35c5841f3 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/cow.py @@ -0,0 +1,136 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Tests for Copy-on-Write (cow.py) +# +# Copyright 2006 Holger Freyther <freyther@handhelds.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import os + +class COWTestCase(unittest.TestCase): + """ + Test case for the COW module from mithro + """ + + def testGetSet(self): + """ + Test and set + """ + from bb.COW import COWDictBase + a = COWDictBase.copy() + + self.assertEquals(False, a.has_key('a')) + + a['a'] = 'a' + a['b'] = 'b' + self.assertEquals(True, a.has_key('a')) + self.assertEquals(True, a.has_key('b')) + self.assertEquals('a', a['a'] ) + self.assertEquals('b', a['b'] ) + + def testCopyCopy(self): + """ + Test the copy of copies + """ + + from bb.COW import COWDictBase + + # create two COW dict 'instances' + b = COWDictBase.copy() + c = COWDictBase.copy() + + # assign some keys to one instance, some keys to another + b['a'] = 10 + b['c'] = 20 + c['a'] = 30 + + # test separation of the two instances + self.assertEquals(False, c.has_key('c')) + self.assertEquals(30, c['a']) + self.assertEquals(10, b['a']) + + # test copy + b_2 = b.copy() + c_2 = c.copy() + + self.assertEquals(False, c_2.has_key('c')) + self.assertEquals(10, b_2['a']) + + b_2['d'] = 40 + self.assertEquals(False, c_2.has_key('d')) + self.assertEquals(True, b_2.has_key('d')) + self.assertEquals(40, b_2['d']) + self.assertEquals(False, b.has_key('d')) + self.assertEquals(False, c.has_key('d')) + + c_2['d'] = 30 + self.assertEquals(True, c_2.has_key('d')) + self.assertEquals(True, b_2.has_key('d')) + self.assertEquals(30, c_2['d']) + self.assertEquals(40, b_2['d']) + self.assertEquals(False, b.has_key('d')) + self.assertEquals(False, c.has_key('d')) + + # test copy of the copy + c_3 = c_2.copy() + b_3 = b_2.copy() + b_3_2 = b_2.copy() + + c_3['e'] = 4711 + self.assertEquals(4711, c_3['e']) + self.assertEquals(False, c_2.has_key('e')) + self.assertEquals(False, b_3.has_key('e')) + self.assertEquals(False, b_3_2.has_key('e')) + self.assertEquals(False, b_2.has_key('e')) + + b_3['e'] = 'viel' + self.assertEquals('viel', b_3['e']) + self.assertEquals(4711, c_3['e']) + self.assertEquals(False, c_2.has_key('e')) + self.assertEquals(True, b_3.has_key('e')) + self.assertEquals(False, b_3_2.has_key('e')) + self.assertEquals(False, b_2.has_key('e')) + + def testCow(self): + from bb.COW import COWDictBase + c = COWDictBase.copy() + c['123'] = 1027 + c['other'] = 4711 + c['d'] = { 'abc' : 10, 'bcd' : 20 } + + copy = c.copy() + + self.assertEquals(1027, c['123']) + self.assertEquals(4711, c['other']) + self.assertEquals({'abc':10, 'bcd':20}, c['d']) + self.assertEquals(1027, copy['123']) + self.assertEquals(4711, copy['other']) + self.assertEquals({'abc':10, 'bcd':20}, copy['d']) + + # cow it now + copy['123'] = 1028 + copy['other'] = 4712 + copy['d']['abc'] = 20 + + + self.assertEquals(1027, c['123']) + self.assertEquals(4711, c['other']) + self.assertEquals({'abc':10, 'bcd':20}, c['d']) + self.assertEquals(1028, copy['123']) + self.assertEquals(4712, copy['other']) + self.assertEquals({'abc':20, 'bcd':20}, copy['d']) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/data.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/data.py new file mode 100644 index 000000000..12232305c --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/data.py @@ -0,0 +1,446 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Tests for the Data Store (data.py/data_smart.py) +# +# Copyright (C) 2010 Chris Larson +# Copyright (C) 2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import bb +import bb.data +import bb.parse +import logging + +class LogRecord(): + def __enter__(self): + logs = [] + class LogHandler(logging.Handler): + def emit(self, record): + logs.append(record) + logger = logging.getLogger("BitBake") + handler = LogHandler() + self.handler = handler + logger.addHandler(handler) + return logs + def __exit__(self, type, value, traceback): + logger = logging.getLogger("BitBake") + logger.removeHandler(self.handler) + return + +def logContains(item, logs): + for l in logs: + m = l.getMessage() + if item in m: + return True + return False + +class DataExpansions(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + self.d["foo"] = "value_of_foo" + self.d["bar"] = "value_of_bar" + self.d["value_of_foo"] = "value_of_'value_of_foo'" + + def test_one_var(self): + val = self.d.expand("${foo}") + self.assertEqual(str(val), "value_of_foo") + + def test_indirect_one_var(self): + val = self.d.expand("${${foo}}") + self.assertEqual(str(val), "value_of_'value_of_foo'") + + def test_indirect_and_another(self): + val = self.d.expand("${${foo}} ${bar}") + self.assertEqual(str(val), "value_of_'value_of_foo' value_of_bar") + + def test_python_snippet(self): + val = self.d.expand("${@5*12}") + self.assertEqual(str(val), "60") + + def test_expand_in_python_snippet(self): + val = self.d.expand("${@'boo ' + '${foo}'}") + self.assertEqual(str(val), "boo value_of_foo") + + def test_python_snippet_getvar(self): + val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}") + self.assertEqual(str(val), "value_of_foo value_of_bar") + + def test_python_unexpanded(self): + self.d.setVar("bar", "${unsetvar}") + val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}") + self.assertEqual(str(val), "${@d.getVar('foo', True) + ' ${unsetvar}'}") + + def test_python_snippet_syntax_error(self): + self.d.setVar("FOO", "${@foo = 5}") + self.assertRaises(bb.data_smart.ExpansionError, self.d.getVar, "FOO", True) + + def test_python_snippet_runtime_error(self): + self.d.setVar("FOO", "${@int('test')}") + self.assertRaises(bb.data_smart.ExpansionError, self.d.getVar, "FOO", True) + + def test_python_snippet_error_path(self): + self.d.setVar("FOO", "foo value ${BAR}") + self.d.setVar("BAR", "bar value ${@int('test')}") + self.assertRaises(bb.data_smart.ExpansionError, self.d.getVar, "FOO", True) + + def test_value_containing_value(self): + val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}") + self.assertEqual(str(val), "value_of_foo value_of_bar") + + def test_reference_undefined_var(self): + val = self.d.expand("${undefinedvar} meh") + self.assertEqual(str(val), "${undefinedvar} meh") + + def test_double_reference(self): + self.d.setVar("BAR", "bar value") + self.d.setVar("FOO", "${BAR} foo ${BAR}") + val = self.d.getVar("FOO", True) + self.assertEqual(str(val), "bar value foo bar value") + + def test_direct_recursion(self): + self.d.setVar("FOO", "${FOO}") + self.assertRaises(bb.data_smart.ExpansionError, self.d.getVar, "FOO", True) + + def test_indirect_recursion(self): + self.d.setVar("FOO", "${BAR}") + self.d.setVar("BAR", "${BAZ}") + self.d.setVar("BAZ", "${FOO}") + self.assertRaises(bb.data_smart.ExpansionError, self.d.getVar, "FOO", True) + + def test_recursion_exception(self): + self.d.setVar("FOO", "${BAR}") + self.d.setVar("BAR", "${${@'FOO'}}") + self.assertRaises(bb.data_smart.ExpansionError, self.d.getVar, "FOO", True) + + def test_incomplete_varexp_single_quotes(self): + self.d.setVar("FOO", "sed -i -e 's:IP{:I${:g' $pc") + val = self.d.getVar("FOO", True) + self.assertEqual(str(val), "sed -i -e 's:IP{:I${:g' $pc") + + def test_nonstring(self): + self.d.setVar("TEST", 5) + val = self.d.getVar("TEST", True) + self.assertEqual(str(val), "5") + + def test_rename(self): + self.d.renameVar("foo", "newfoo") + self.assertEqual(self.d.getVar("newfoo", False), "value_of_foo") + self.assertEqual(self.d.getVar("foo", False), None) + + def test_deletion(self): + self.d.delVar("foo") + self.assertEqual(self.d.getVar("foo", False), None) + + def test_keys(self): + keys = self.d.keys() + self.assertEqual(keys, ['value_of_foo', 'foo', 'bar']) + + def test_keys_deletion(self): + newd = bb.data.createCopy(self.d) + newd.delVar("bar") + keys = newd.keys() + self.assertEqual(keys, ['value_of_foo', 'foo']) + +class TestNestedExpansions(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + self.d["foo"] = "foo" + self.d["bar"] = "bar" + self.d["value_of_foobar"] = "187" + + def test_refs(self): + val = self.d.expand("${value_of_${foo}${bar}}") + self.assertEqual(str(val), "187") + + #def test_python_refs(self): + # val = self.d.expand("${@${@3}**2 + ${@4}**2}") + # self.assertEqual(str(val), "25") + + def test_ref_in_python_ref(self): + val = self.d.expand("${@'${foo}' + 'bar'}") + self.assertEqual(str(val), "foobar") + + def test_python_ref_in_ref(self): + val = self.d.expand("${${@'f'+'o'+'o'}}") + self.assertEqual(str(val), "foo") + + def test_deep_nesting(self): + depth = 100 + val = self.d.expand("${" * depth + "foo" + "}" * depth) + self.assertEqual(str(val), "foo") + + #def test_deep_python_nesting(self): + # depth = 50 + # val = self.d.expand("${@" * depth + "1" + "+1}" * depth) + # self.assertEqual(str(val), str(depth + 1)) + + def test_mixed(self): + val = self.d.expand("${value_of_${@('${foo}'+'bar')[0:3]}${${@'BAR'.lower()}}}") + self.assertEqual(str(val), "187") + + def test_runtime(self): + val = self.d.expand("${${@'value_of' + '_f'+'o'+'o'+'b'+'a'+'r'}}") + self.assertEqual(str(val), "187") + +class TestMemoize(unittest.TestCase): + def test_memoized(self): + d = bb.data.init() + d.setVar("FOO", "bar") + self.assertTrue(d.getVar("FOO", False) is d.getVar("FOO", False)) + + def test_not_memoized(self): + d1 = bb.data.init() + d2 = bb.data.init() + d1.setVar("FOO", "bar") + d2.setVar("FOO", "bar2") + self.assertTrue(d1.getVar("FOO", False) is not d2.getVar("FOO", False)) + + def test_changed_after_memoized(self): + d = bb.data.init() + d.setVar("foo", "value of foo") + self.assertEqual(str(d.getVar("foo", False)), "value of foo") + d.setVar("foo", "second value of foo") + self.assertEqual(str(d.getVar("foo", False)), "second value of foo") + + def test_same_value(self): + d = bb.data.init() + d.setVar("foo", "value of") + d.setVar("bar", "value of") + self.assertEqual(d.getVar("foo", False), + d.getVar("bar", False)) + +class TestConcat(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + self.d.setVar("FOO", "foo") + self.d.setVar("VAL", "val") + self.d.setVar("BAR", "bar") + + def test_prepend(self): + self.d.setVar("TEST", "${VAL}") + self.d.prependVar("TEST", "${FOO}:") + self.assertEqual(self.d.getVar("TEST", True), "foo:val") + + def test_append(self): + self.d.setVar("TEST", "${VAL}") + self.d.appendVar("TEST", ":${BAR}") + self.assertEqual(self.d.getVar("TEST", True), "val:bar") + + def test_multiple_append(self): + self.d.setVar("TEST", "${VAL}") + self.d.prependVar("TEST", "${FOO}:") + self.d.appendVar("TEST", ":val2") + self.d.appendVar("TEST", ":${BAR}") + self.assertEqual(self.d.getVar("TEST", True), "foo:val:val2:bar") + +class TestConcatOverride(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + self.d.setVar("FOO", "foo") + self.d.setVar("VAL", "val") + self.d.setVar("BAR", "bar") + + def test_prepend(self): + self.d.setVar("TEST", "${VAL}") + self.d.setVar("TEST_prepend", "${FOO}:") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "foo:val") + + def test_append(self): + self.d.setVar("TEST", "${VAL}") + self.d.setVar("TEST_append", ":${BAR}") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "val:bar") + + def test_multiple_append(self): + self.d.setVar("TEST", "${VAL}") + self.d.setVar("TEST_prepend", "${FOO}:") + self.d.setVar("TEST_append", ":val2") + self.d.setVar("TEST_append", ":${BAR}") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "foo:val:val2:bar") + + def test_append_unset(self): + self.d.setVar("TEST_prepend", "${FOO}:") + self.d.setVar("TEST_append", ":val2") + self.d.setVar("TEST_append", ":${BAR}") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "foo::val2:bar") + + def test_remove(self): + self.d.setVar("TEST", "${VAL} ${BAR}") + self.d.setVar("TEST_remove", "val") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "bar") + + def test_doubleref_remove(self): + self.d.setVar("TEST", "${VAL} ${BAR}") + self.d.setVar("TEST_remove", "val") + self.d.setVar("TEST_TEST", "${TEST} ${TEST}") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST_TEST", True), "bar bar") + + def test_empty_remove(self): + self.d.setVar("TEST", "") + self.d.setVar("TEST_remove", "val") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "") + + def test_remove_expansion(self): + self.d.setVar("BAR", "Z") + self.d.setVar("TEST", "${BAR}/X Y") + self.d.setVar("TEST_remove", "${BAR}/X") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "Y") + + def test_remove_expansion_items(self): + self.d.setVar("TEST", "A B C D") + self.d.setVar("BAR", "B D") + self.d.setVar("TEST_remove", "${BAR}") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "A C") + +class TestOverrides(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + self.d.setVar("OVERRIDES", "foo:bar:local") + self.d.setVar("TEST", "testvalue") + + def test_no_override(self): + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "testvalue") + + def test_one_override(self): + self.d.setVar("TEST_bar", "testvalue2") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "testvalue2") + + def test_one_override_unset(self): + self.d.setVar("TEST2_bar", "testvalue2") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST2", True), "testvalue2") + self.assertItemsEqual(self.d.keys(), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2_bar']) + + def test_multiple_override(self): + self.d.setVar("TEST_bar", "testvalue2") + self.d.setVar("TEST_local", "testvalue3") + self.d.setVar("TEST_foo", "testvalue4") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "testvalue3") + self.assertItemsEqual(self.d.keys(), ['TEST', 'TEST_foo', 'OVERRIDES', 'TEST_bar', 'TEST_local']) + + def test_multiple_combined_overrides(self): + self.d.setVar("TEST_local_foo_bar", "testvalue3") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "testvalue3") + + def test_multiple_overrides_unset(self): + self.d.setVar("TEST2_local_foo_bar", "testvalue3") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST2", True), "testvalue3") + + def test_keyexpansion_override(self): + self.d.setVar("LOCAL", "local") + self.d.setVar("TEST_bar", "testvalue2") + self.d.setVar("TEST_${LOCAL}", "testvalue3") + self.d.setVar("TEST_foo", "testvalue4") + bb.data.update_data(self.d) + bb.data.expandKeys(self.d) + self.assertEqual(self.d.getVar("TEST", True), "testvalue3") + + def test_rename_override(self): + self.d.setVar("ALTERNATIVE_ncurses-tools_class-target", "a") + self.d.setVar("OVERRIDES", "class-target") + bb.data.update_data(self.d) + self.d.renameVar("ALTERNATIVE_ncurses-tools", "ALTERNATIVE_lib32-ncurses-tools") + self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools", True), "a") + + def test_underscore_override(self): + self.d.setVar("TEST_bar", "testvalue2") + self.d.setVar("TEST_some_val", "testvalue3") + self.d.setVar("TEST_foo", "testvalue4") + self.d.setVar("OVERRIDES", "foo:bar:some_val") + self.assertEqual(self.d.getVar("TEST", True), "testvalue3") + +class TestKeyExpansion(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + self.d.setVar("FOO", "foo") + self.d.setVar("BAR", "foo") + + def test_keyexpand(self): + self.d.setVar("VAL_${FOO}", "A") + self.d.setVar("VAL_${BAR}", "B") + with LogRecord() as logs: + bb.data.expandKeys(self.d) + self.assertTrue(logContains("Variable key VAL_${FOO} (A) replaces original key VAL_foo (B)", logs)) + self.assertEqual(self.d.getVar("VAL_foo", True), "A") + +class TestFlags(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + self.d.setVar("foo", "value of foo") + self.d.setVarFlag("foo", "flag1", "value of flag1") + self.d.setVarFlag("foo", "flag2", "value of flag2") + + def test_setflag(self): + self.assertEqual(self.d.getVarFlag("foo", "flag1", False), "value of flag1") + self.assertEqual(self.d.getVarFlag("foo", "flag2", False), "value of flag2") + + def test_delflag(self): + self.d.delVarFlag("foo", "flag2") + self.assertEqual(self.d.getVarFlag("foo", "flag1", False), "value of flag1") + self.assertEqual(self.d.getVarFlag("foo", "flag2", False), None) + + +class Contains(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + self.d.setVar("SOMEFLAG", "a b c") + + def test_contains(self): + self.assertTrue(bb.utils.contains("SOMEFLAG", "a", True, False, self.d)) + self.assertTrue(bb.utils.contains("SOMEFLAG", "b", True, False, self.d)) + self.assertTrue(bb.utils.contains("SOMEFLAG", "c", True, False, self.d)) + + self.assertTrue(bb.utils.contains("SOMEFLAG", "a b", True, False, self.d)) + self.assertTrue(bb.utils.contains("SOMEFLAG", "b c", True, False, self.d)) + self.assertTrue(bb.utils.contains("SOMEFLAG", "c a", True, False, self.d)) + + self.assertTrue(bb.utils.contains("SOMEFLAG", "a b c", True, False, self.d)) + self.assertTrue(bb.utils.contains("SOMEFLAG", "c b a", True, False, self.d)) + + self.assertFalse(bb.utils.contains("SOMEFLAG", "x", True, False, self.d)) + self.assertFalse(bb.utils.contains("SOMEFLAG", "a x", True, False, self.d)) + self.assertFalse(bb.utils.contains("SOMEFLAG", "x c b", True, False, self.d)) + self.assertFalse(bb.utils.contains("SOMEFLAG", "x c b a", True, False, self.d)) + + def test_contains_any(self): + self.assertTrue(bb.utils.contains_any("SOMEFLAG", "a", True, False, self.d)) + self.assertTrue(bb.utils.contains_any("SOMEFLAG", "b", True, False, self.d)) + self.assertTrue(bb.utils.contains_any("SOMEFLAG", "c", True, False, self.d)) + + self.assertTrue(bb.utils.contains_any("SOMEFLAG", "a b", True, False, self.d)) + self.assertTrue(bb.utils.contains_any("SOMEFLAG", "b c", True, False, self.d)) + self.assertTrue(bb.utils.contains_any("SOMEFLAG", "c a", True, False, self.d)) + + self.assertTrue(bb.utils.contains_any("SOMEFLAG", "a x", True, False, self.d)) + self.assertTrue(bb.utils.contains_any("SOMEFLAG", "x c", True, False, self.d)) + + self.assertFalse(bb.utils.contains_any("SOMEFLAG", "x", True, False, self.d)) + self.assertFalse(bb.utils.contains_any("SOMEFLAG", "x y z", True, False, self.d)) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py new file mode 100644 index 000000000..4ba688bfe --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py @@ -0,0 +1,812 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Tests for the Fetcher (fetch2/) +# +# Copyright (C) 2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import tempfile +import subprocess +import os +from bb.fetch2 import URI +from bb.fetch2 import FetchMethod +import bb + +class URITest(unittest.TestCase): + test_uris = { + "http://www.google.com/index.html" : { + 'uri': 'http://www.google.com/index.html', + 'scheme': 'http', + 'hostname': 'www.google.com', + 'port': None, + 'hostport': 'www.google.com', + 'path': '/index.html', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': {}, + 'relative': False + }, + "http://www.google.com/index.html;param1=value1" : { + 'uri': 'http://www.google.com/index.html;param1=value1', + 'scheme': 'http', + 'hostname': 'www.google.com', + 'port': None, + 'hostport': 'www.google.com', + 'path': '/index.html', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': { + 'param1': 'value1' + }, + 'query': {}, + 'relative': False + }, + "http://www.example.org/index.html?param1=value1" : { + 'uri': 'http://www.example.org/index.html?param1=value1', + 'scheme': 'http', + 'hostname': 'www.example.org', + 'port': None, + 'hostport': 'www.example.org', + 'path': '/index.html', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': { + 'param1': 'value1' + }, + 'relative': False + }, + "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : { + 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', + 'scheme': 'http', + 'hostname': 'www.example.org', + 'port': None, + 'hostport': 'www.example.org', + 'path': '/index.html', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': { + 'param2': 'value2' + }, + 'query': { + 'qparam1': 'qvalue1' + }, + 'relative': False + }, + "http://www.example.com:8080/index.html" : { + 'uri': 'http://www.example.com:8080/index.html', + 'scheme': 'http', + 'hostname': 'www.example.com', + 'port': 8080, + 'hostport': 'www.example.com:8080', + 'path': '/index.html', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': {}, + 'relative': False + }, + "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { + 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', + 'scheme': 'cvs', + 'hostname': 'cvs.handhelds.org', + 'port': None, + 'hostport': 'cvs.handhelds.org', + 'path': '/cvs', + 'userinfo': 'anoncvs', + 'username': 'anoncvs', + 'password': '', + 'params': { + 'module': 'familiar/dist/ipkg' + }, + 'query': {}, + 'relative': False + }, + "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { + 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', + 'scheme': 'cvs', + 'hostname': 'cvs.handhelds.org', + 'port': None, + 'hostport': 'cvs.handhelds.org', + 'path': '/cvs', + 'userinfo': 'anoncvs:anonymous', + 'username': 'anoncvs', + 'password': 'anonymous', + 'params': { + 'tag': 'V0-99-81', + 'module': 'familiar/dist/ipkg' + }, + 'query': {}, + 'relative': False + }, + "file://example.diff": { # NOTE: Not RFC compliant! + 'uri': 'file:example.diff', + 'scheme': 'file', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': 'example.diff', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': {}, + 'relative': True + }, + "file:example.diff": { # NOTE: RFC compliant version of the former + 'uri': 'file:example.diff', + 'scheme': 'file', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': 'example.diff', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': {}, + 'relative': True + }, + "file:///tmp/example.diff": { + 'uri': 'file:///tmp/example.diff', + 'scheme': 'file', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': '/tmp/example.diff', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': {}, + 'relative': False + }, + "git:///path/example.git": { + 'uri': 'git:///path/example.git', + 'scheme': 'git', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': '/path/example.git', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': {}, + 'relative': False + }, + "git:path/example.git": { + 'uri': 'git:path/example.git', + 'scheme': 'git', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': 'path/example.git', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': {}, + 'relative': True + }, + "git://example.net/path/example.git": { + 'uri': 'git://example.net/path/example.git', + 'scheme': 'git', + 'hostname': 'example.net', + 'port': None, + 'hostport': 'example.net', + 'path': '/path/example.git', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': {}, + 'relative': False + }, + "http://somesite.net;someparam=1": { + 'uri': 'http://somesite.net;someparam=1', + 'scheme': 'http', + 'hostname': 'somesite.net', + 'port': None, + 'hostport': 'somesite.net', + 'path': '', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {"someparam" : "1"}, + 'query': {}, + 'relative': False + }, + "file://somelocation;someparam=1": { + 'uri': 'file:somelocation;someparam=1', + 'scheme': 'file', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': 'somelocation', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {"someparam" : "1"}, + 'query': {}, + 'relative': True + } + + } + + def test_uri(self): + for test_uri, ref in self.test_uris.items(): + uri = URI(test_uri) + + self.assertEqual(str(uri), ref['uri']) + + # expected attributes + self.assertEqual(uri.scheme, ref['scheme']) + + self.assertEqual(uri.userinfo, ref['userinfo']) + self.assertEqual(uri.username, ref['username']) + self.assertEqual(uri.password, ref['password']) + + self.assertEqual(uri.hostname, ref['hostname']) + self.assertEqual(uri.port, ref['port']) + self.assertEqual(uri.hostport, ref['hostport']) + + self.assertEqual(uri.path, ref['path']) + self.assertEqual(uri.params, ref['params']) + + self.assertEqual(uri.relative, ref['relative']) + + def test_dict(self): + for test in self.test_uris.values(): + uri = URI() + + self.assertEqual(uri.scheme, '') + self.assertEqual(uri.userinfo, '') + self.assertEqual(uri.username, '') + self.assertEqual(uri.password, '') + self.assertEqual(uri.hostname, '') + self.assertEqual(uri.port, None) + self.assertEqual(uri.path, '') + self.assertEqual(uri.params, {}) + + + uri.scheme = test['scheme'] + self.assertEqual(uri.scheme, test['scheme']) + + uri.userinfo = test['userinfo'] + self.assertEqual(uri.userinfo, test['userinfo']) + self.assertEqual(uri.username, test['username']) + self.assertEqual(uri.password, test['password']) + + # make sure changing the values doesn't do anything unexpected + uri.username = 'changeme' + self.assertEqual(uri.username, 'changeme') + self.assertEqual(uri.password, test['password']) + uri.password = 'insecure' + self.assertEqual(uri.username, 'changeme') + self.assertEqual(uri.password, 'insecure') + + # reset back after our trickery + uri.userinfo = test['userinfo'] + self.assertEqual(uri.userinfo, test['userinfo']) + self.assertEqual(uri.username, test['username']) + self.assertEqual(uri.password, test['password']) + + uri.hostname = test['hostname'] + self.assertEqual(uri.hostname, test['hostname']) + self.assertEqual(uri.hostport, test['hostname']) + + uri.port = test['port'] + self.assertEqual(uri.port, test['port']) + self.assertEqual(uri.hostport, test['hostport']) + + uri.path = test['path'] + self.assertEqual(uri.path, test['path']) + + uri.params = test['params'] + self.assertEqual(uri.params, test['params']) + + uri.query = test['query'] + self.assertEqual(uri.query, test['query']) + + self.assertEqual(str(uri), test['uri']) + + uri.params = {} + self.assertEqual(uri.params, {}) + self.assertEqual(str(uri), (str(uri).split(";"))[0]) + +class FetcherTest(unittest.TestCase): + + def setUp(self): + self.origdir = os.getcwd() + self.d = bb.data.init() + self.tempdir = tempfile.mkdtemp() + self.dldir = os.path.join(self.tempdir, "download") + os.mkdir(self.dldir) + self.d.setVar("DL_DIR", self.dldir) + self.unpackdir = os.path.join(self.tempdir, "unpacked") + os.mkdir(self.unpackdir) + persistdir = os.path.join(self.tempdir, "persistdata") + self.d.setVar("PERSISTENT_DIR", persistdir) + + def tearDown(self): + os.chdir(self.origdir) + bb.utils.prunedir(self.tempdir) + +class MirrorUriTest(FetcherTest): + + replaceuris = { + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") + : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") + : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") + : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", + ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") + : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", + ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") + : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", + ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") + : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", + ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") + : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", + ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") + : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", + ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") + : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") + : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + + #Renaming files doesn't work + #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" + #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", + } + + mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \ + "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \ + "https://.*/.* file:///someotherpath/downloads/ \n" \ + "http://.*/.* file:///someotherpath/downloads/ \n" + + def test_urireplace(self): + for k, v in self.replaceuris.items(): + ud = bb.fetch.FetchData(k[0], self.d) + ud.setup_localpath(self.d) + mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) + newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) + self.assertEqual([v], newuris) + + def test_urilist1(self): + fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) + mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) + uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) + self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) + + def test_urilist2(self): + # Catch https:// -> files:// bug + fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) + mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) + uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) + self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) + + def test_mirror_of_mirror(self): + # Test if mirror of a mirror works + mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n" + mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n" + fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) + mirrors = bb.fetch2.mirror_from_string(mirrorvar) + uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) + self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', + 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', + 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', + 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) + + recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \ + "https://.*/[^/]* https://BBBB/B/B/B/ \n" + + def test_recursive(self): + fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) + mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) + uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) + self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz', + 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz', + 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) + +class FetcherLocalTest(FetcherTest): + def setUp(self): + def touch(fn): + with file(fn, 'a'): + os.utime(fn, None) + + super(FetcherLocalTest, self).setUp() + self.localsrcdir = os.path.join(self.tempdir, 'localsrc') + os.makedirs(self.localsrcdir) + touch(os.path.join(self.localsrcdir, 'a')) + touch(os.path.join(self.localsrcdir, 'b')) + os.makedirs(os.path.join(self.localsrcdir, 'dir')) + touch(os.path.join(self.localsrcdir, 'dir', 'c')) + touch(os.path.join(self.localsrcdir, 'dir', 'd')) + os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) + touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) + self.d.setVar("FILESPATH", self.localsrcdir) + + def fetchUnpack(self, uris): + fetcher = bb.fetch.Fetch(uris, self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + flst = [] + for root, dirs, files in os.walk(self.unpackdir): + for f in files: + flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir)) + flst.sort() + return flst + + def test_local(self): + tree = self.fetchUnpack(['file://a', 'file://dir/c']) + self.assertEqual(tree, ['a', 'dir/c']) + + def test_local_wildcard(self): + tree = self.fetchUnpack(['file://a', 'file://dir/*']) + self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) + + def test_local_dir(self): + tree = self.fetchUnpack(['file://a', 'file://dir']) + self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) + + def test_local_subdir(self): + tree = self.fetchUnpack(['file://dir/subdir']) + self.assertEqual(tree, ['dir/subdir/e']) + + def test_local_subdir_file(self): + tree = self.fetchUnpack(['file://dir/subdir/e']) + self.assertEqual(tree, ['dir/subdir/e']) + + def test_local_subdirparam(self): + tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo']) + self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e']) + + def test_local_deepsubdirparam(self): + tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar']) + self.assertEqual(tree, ['bar/dir/subdir/e']) + +class FetcherNetworkTest(FetcherTest): + + if os.environ.get("BB_SKIP_NETTESTS") == "yes": + print("Unset BB_SKIP_NETTESTS to run network tests") + else: + def test_fetch(self): + fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) + self.d.setVar("BB_NO_NETWORK", "1") + fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) + self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) + + def test_fetch_mirror(self): + self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake") + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) + + def test_fetch_mirror_of_mirror(self): + self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake") + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) + + def test_fetch_file_mirror_of_mirror(self): + self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake") + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) + os.mkdir(self.dldir + "/some2where") + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) + + def test_fetch_premirror(self): + self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake") + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) + + def gitfetcher(self, url1, url2): + def checkrevision(self, fetcher): + fetcher.unpack(self.unpackdir) + revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip() + self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") + + self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") + self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") + fetcher = bb.fetch.Fetch([url1], self.d) + fetcher.download() + checkrevision(self, fetcher) + # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works + bb.utils.prunedir(self.dldir + "/git2/") + bb.utils.prunedir(self.unpackdir) + self.d.setVar("BB_NO_NETWORK", "1") + fetcher = bb.fetch.Fetch([url2], self.d) + fetcher.download() + checkrevision(self, fetcher) + + def test_gitfetch(self): + url1 = url2 = "git://git.openembedded.org/bitbake" + self.gitfetcher(url1, url2) + + def test_gitfetch_goodsrcrev(self): + # SRCREV is set but matches rev= parameter + url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5" + self.gitfetcher(url1, url2) + + def test_gitfetch_badsrcrev(self): + # SRCREV is set but does not match rev= parameter + url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5" + self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) + + def test_gitfetch_tagandrev(self): + # SRCREV is set but does not match rev= parameter + url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5" + self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) + + def test_gitfetch_premirror(self): + url1 = "git://git.openembedded.org/bitbake" + url2 = "git://someserver.org/bitbake" + self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") + self.gitfetcher(url1, url2) + + def test_gitfetch_premirror2(self): + url1 = url2 = "git://someserver.org/bitbake" + self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") + self.gitfetcher(url1, url2) + + def test_gitfetch_premirror3(self): + realurl = "git://git.openembedded.org/bitbake" + dummyurl = "git://someserver.org/bitbake" + self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") + os.chdir(self.tempdir) + bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True) + self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir)) + self.gitfetcher(dummyurl, dummyurl) + + def test_git_submodule(self): + fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d) + fetcher.download() + # Previous cwd has been deleted + os.chdir(os.path.dirname(self.unpackdir)) + fetcher.unpack(self.unpackdir) + + +class TrustedNetworksTest(FetcherTest): + def test_trusted_network(self): + # Ensure trusted_network returns False when the host IS in the list. + url = "git://Someserver.org/foo;rev=1" + self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org") + self.assertTrue(bb.fetch.trusted_network(self.d, url)) + + def test_wild_trusted_network(self): + # Ensure trusted_network returns true when the *.host IS in the list. + url = "git://Someserver.org/foo;rev=1" + self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") + self.assertTrue(bb.fetch.trusted_network(self.d, url)) + + def test_prefix_wild_trusted_network(self): + # Ensure trusted_network returns true when the prefix matches *.host. + url = "git://git.Someserver.org/foo;rev=1" + self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") + self.assertTrue(bb.fetch.trusted_network(self.d, url)) + + def test_two_prefix_wild_trusted_network(self): + # Ensure trusted_network returns true when the prefix matches *.host. + url = "git://something.git.Someserver.org/foo;rev=1" + self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") + self.assertTrue(bb.fetch.trusted_network(self.d, url)) + + def test_port_trusted_network(self): + # Ensure trusted_network returns True, even if the url specifies a port. + url = "git://someserver.org:8080/foo;rev=1" + self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org") + self.assertTrue(bb.fetch.trusted_network(self.d, url)) + + def test_untrusted_network(self): + # Ensure trusted_network returns False when the host is NOT in the list. + url = "git://someserver.org/foo;rev=1" + self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") + self.assertFalse(bb.fetch.trusted_network(self.d, url)) + + def test_wild_untrusted_network(self): + # Ensure trusted_network returns False when the host is NOT in the list. + url = "git://*.someserver.org/foo;rev=1" + self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") + self.assertFalse(bb.fetch.trusted_network(self.d, url)) + +class URLHandle(unittest.TestCase): + + datatable = { + "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), + "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), + "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}), + "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}), + "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), + } + # we require a pathname to encodeurl but users can still pass such urls to + # decodeurl and we need to handle them + decodedata = datatable.copy() + decodedata.update({ + "http://somesite.net;someparam=1": ('http', 'somesite.net', '', '', '', {'someparam': '1'}), + }) + + def test_decodeurl(self): + for k, v in self.decodedata.items(): + result = bb.fetch.decodeurl(k) + self.assertEqual(result, v) + + def test_encodeurl(self): + for k, v in self.datatable.items(): + result = bb.fetch.encodeurl(v) + self.assertEqual(result, k) + +class FetchLatestVersionTest(FetcherTest): + + test_git_uris = { + # version pattern "X.Y.Z" + ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "") + : "1.99.4", + # version pattern "vX.Y" + ("mtd-utils", "git://git.infradead.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") + : "1.5.0", + # version pattern "pkg_name-X.Y" + ("presentproto", "git://anongit.freedesktop.org/git/xorg/proto/presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "") + : "1.0", + # version pattern "pkg_name-vX.Y.Z" + ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "") + : "1.4.0", + # combination version pattern + ("sysprof", "git://git.gnome.org/sysprof", "cd44ee6644c3641507fb53b8a2a69137f2971219", "") + : "1.2.0", + ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "") + : "2014.01", + # version pattern "yyyymmdd" + ("mobile-broadband-provider-info", "git://git.gnome.org/mobile-broadband-provider-info", "4ed19e11c2975105b71b956440acdb25d46a347d", "") + : "20120614", + # packages with a valid UPSTREAM_CHECK_GITTAGREGEX + ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))") + : "0.4.3", + ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))") + : "11.0.0", + ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))") + : "1.3.59", + ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))") + : "3.82+dbg0.9", + } + + test_wget_uris = { + # packages with versions inside directory name + ("util-linux", "http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "") + : "2.24.2", + ("enchant", "http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "") + : "1.6.0", + ("cmake", "http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") + : "2.8.12.1", + # packages with versions only in current directory + ("eglic", "http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") + : "2.19", + ("gnu-config", "http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") + : "20120814", + # packages with "99" in the name of possible version + ("pulseaudio", "http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "") + : "5.0", + ("xserver-xorg", "http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "") + : "1.15.1", + # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX + ("cups", "http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2", "https://github.com/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") + : "2.0.0", + ("db", "http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz", "http://www.oracle.com/technetwork/products/berkeleydb/downloads/index-082944.html", "http://download.oracle.com/otn/berkeley-db/(?P<name>db-)(?P<pver>((\d+[\.\-_]*)+))\.tar\.gz") + : "6.1.19", + } + if os.environ.get("BB_SKIP_NETTESTS") == "yes": + print("Unset BB_SKIP_NETTESTS to run network tests") + else: + def test_git_latest_versionstring(self): + for k, v in self.test_git_uris.items(): + self.d.setVar("PN", k[0]) + self.d.setVar("SRCREV", k[2]) + self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3]) + ud = bb.fetch2.FetchData(k[1], self.d) + pupver= ud.method.latest_versionstring(ud, self.d) + verstring = pupver[0] + r = bb.utils.vercmp_string(v, verstring) + self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) + + def test_wget_latest_versionstring(self): + for k, v in self.test_wget_uris.items(): + self.d.setVar("PN", k[0]) + self.d.setVar("UPSTREAM_CHECK_URI", k[2]) + self.d.setVar("UPSTREAM_CHECK_REGEX", k[3]) + ud = bb.fetch2.FetchData(k[1], self.d) + pupver = ud.method.latest_versionstring(ud, self.d) + verstring = pupver[0] + r = bb.utils.vercmp_string(v, verstring) + self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) + + +class FetchCheckStatusTest(FetcherTest): + test_wget_uris = ["http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2", + "http://www.cups.org/software/ipptool/ipptool-20130731-linux-ubuntu-i686.tar.gz", + "http://www.cups.org/", + "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", + "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", + "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", + "https://yoctoproject.org/", + "https://yoctoproject.org/documentation", + "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", + "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", + "ftp://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz", + "ftp://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz", + "ftp://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz", + # GitHub releases are hosted on Amazon S3, which doesn't support HEAD + "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" + ] + + if os.environ.get("BB_SKIP_NETTESTS") == "yes": + print("Unset BB_SKIP_NETTESTS to run network tests") + else: + + def test_wget_checkstatus(self): + fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d) + for u in self.test_wget_uris: + ud = fetch.ud[u] + m = ud.method + ret = m.checkstatus(fetch, ud, self.d) + self.assertTrue(ret, msg="URI %s, can't check status" % (u)) + + + def test_wget_checkstatus_connection_cache(self): + from bb.fetch2 import FetchConnectionCache + + connection_cache = FetchConnectionCache() + fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d, + connection_cache = connection_cache) + + for u in self.test_wget_uris: + ud = fetch.ud[u] + m = ud.method + ret = m.checkstatus(fetch, ud, self.d) + self.assertTrue(ret, msg="URI %s, can't check status" % (u)) + + connection_cache.close_connections() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/parse.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/parse.py new file mode 100644 index 000000000..6beb76a48 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/parse.py @@ -0,0 +1,147 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Test for lib/bb/parse/ +# +# Copyright (C) 2015 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import tempfile +import logging +import bb +import os + +logger = logging.getLogger('BitBake.TestParse') + +import bb.parse +import bb.data +import bb.siggen + +class ParseTest(unittest.TestCase): + + testfile = """ +A = "1" +B = "2" +do_install() { + echo "hello" +} + +C = "3" +""" + + def setUp(self): + self.d = bb.data.init() + bb.parse.siggen = bb.siggen.init(self.d) + + def parsehelper(self, content, suffix = ".bb"): + + f = tempfile.NamedTemporaryFile(suffix = suffix) + f.write(content) + f.flush() + os.chdir(os.path.dirname(f.name)) + return f + + def test_parse_simple(self): + f = self.parsehelper(self.testfile) + d = bb.parse.handle(f.name, self.d)[''] + self.assertEqual(d.getVar("A", True), "1") + self.assertEqual(d.getVar("B", True), "2") + self.assertEqual(d.getVar("C", True), "3") + + def test_parse_incomplete_function(self): + testfileB = self.testfile.replace("}", "") + f = self.parsehelper(testfileB) + with self.assertRaises(bb.parse.ParseError): + d = bb.parse.handle(f.name, self.d)[''] + + overridetest = """ +RRECOMMENDS_${PN} = "a" +RRECOMMENDS_${PN}_libc = "b" +OVERRIDES = "libc:${PN}" +PN = "gtk+" +""" + + def test_parse_overrides(self): + f = self.parsehelper(self.overridetest) + d = bb.parse.handle(f.name, self.d)[''] + self.assertEqual(d.getVar("RRECOMMENDS", True), "b") + bb.data.expandKeys(d) + self.assertEqual(d.getVar("RRECOMMENDS", True), "b") + d.setVar("RRECOMMENDS_gtk+", "c") + self.assertEqual(d.getVar("RRECOMMENDS", True), "c") + + overridetest2 = """ +EXTRA_OECONF = "" +EXTRA_OECONF_class-target = "b" +EXTRA_OECONF_append = " c" +""" + + def test_parse_overrides(self): + f = self.parsehelper(self.overridetest2) + d = bb.parse.handle(f.name, self.d)[''] + d.appendVar("EXTRA_OECONF", " d") + d.setVar("OVERRIDES", "class-target") + self.assertEqual(d.getVar("EXTRA_OECONF", True), "b c d") + + overridetest3 = """ +DESCRIPTION = "A" +DESCRIPTION_${PN}-dev = "${DESCRIPTION} B" +PN = "bc" +""" + + def test_parse_combinations(self): + f = self.parsehelper(self.overridetest3) + d = bb.parse.handle(f.name, self.d)[''] + bb.data.expandKeys(d) + self.assertEqual(d.getVar("DESCRIPTION_bc-dev", True), "A B") + d.setVar("DESCRIPTION", "E") + d.setVar("DESCRIPTION_bc-dev", "C D") + d.setVar("OVERRIDES", "bc-dev") + self.assertEqual(d.getVar("DESCRIPTION", True), "C D") + + + classextend = """ +VAR_var_override1 = "B" +EXTRA = ":override1" +OVERRIDES = "nothing${EXTRA}" + +BBCLASSEXTEND = "###CLASS###" +""" + classextend_bbclass = """ +EXTRA = "" +python () { + d.renameVar("VAR_var", "VAR_var2") +} +""" + + # + # Test based upon a real world data corruption issue. One + # data store changing a variable poked through into a different data + # store. This test case replicates that issue where the value 'B' would + # become unset/disappear. + # + def test_parse_classextend_contamination(self): + cls = self.parsehelper(self.classextend_bbclass, suffix=".bbclass") + #clsname = os.path.basename(cls.name).replace(".bbclass", "") + self.classextend = self.classextend.replace("###CLASS###", cls.name) + f = self.parsehelper(self.classextend) + alldata = bb.parse.handle(f.name, self.d) + d1 = alldata[''] + d2 = alldata[cls.name] + self.assertEqual(d1.getVar("VAR_var", True), "B") + self.assertEqual(d2.getVar("VAR_var", True), None) + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/utils.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/utils.py new file mode 100644 index 000000000..2f4ccf3c6 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/utils.py @@ -0,0 +1,603 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Tests for utils.py +# +# Copyright (C) 2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import bb +import os +import tempfile +import re + +class VerCmpString(unittest.TestCase): + + def test_vercmpstring(self): + result = bb.utils.vercmp_string('1', '2') + self.assertTrue(result < 0) + result = bb.utils.vercmp_string('2', '1') + self.assertTrue(result > 0) + result = bb.utils.vercmp_string('1', '1.0') + self.assertTrue(result < 0) + result = bb.utils.vercmp_string('1', '1.1') + self.assertTrue(result < 0) + result = bb.utils.vercmp_string('1.1', '1_p2') + self.assertTrue(result < 0) + result = bb.utils.vercmp_string('1.0', '1.0+1.1-beta1') + self.assertTrue(result < 0) + result = bb.utils.vercmp_string('1.1', '1.0+1.1-beta1') + self.assertTrue(result > 0) + + def test_explode_dep_versions(self): + correctresult = {"foo" : ["= 1.10"]} + result = bb.utils.explode_dep_versions2("foo (= 1.10)") + self.assertEqual(result, correctresult) + result = bb.utils.explode_dep_versions2("foo (=1.10)") + self.assertEqual(result, correctresult) + result = bb.utils.explode_dep_versions2("foo ( = 1.10)") + self.assertEqual(result, correctresult) + result = bb.utils.explode_dep_versions2("foo ( =1.10)") + self.assertEqual(result, correctresult) + result = bb.utils.explode_dep_versions2("foo ( = 1.10 )") + self.assertEqual(result, correctresult) + result = bb.utils.explode_dep_versions2("foo ( =1.10 )") + self.assertEqual(result, correctresult) + + def test_vercmp_string_op(self): + compareops = [('1', '1', '=', True), + ('1', '1', '==', True), + ('1', '1', '!=', False), + ('1', '1', '>', False), + ('1', '1', '<', False), + ('1', '1', '>=', True), + ('1', '1', '<=', True), + ('1', '0', '=', False), + ('1', '0', '==', False), + ('1', '0', '!=', True), + ('1', '0', '>', True), + ('1', '0', '<', False), + ('1', '0', '>>', True), + ('1', '0', '<<', False), + ('1', '0', '>=', True), + ('1', '0', '<=', False), + ('0', '1', '=', False), + ('0', '1', '==', False), + ('0', '1', '!=', True), + ('0', '1', '>', False), + ('0', '1', '<', True), + ('0', '1', '>>', False), + ('0', '1', '<<', True), + ('0', '1', '>=', False), + ('0', '1', '<=', True)] + + for arg1, arg2, op, correctresult in compareops: + result = bb.utils.vercmp_string_op(arg1, arg2, op) + self.assertEqual(result, correctresult, 'vercmp_string_op("%s", "%s", "%s") != %s' % (arg1, arg2, op, correctresult)) + + # Check that clearly invalid operator raises an exception + self.assertRaises(bb.utils.VersionStringException, bb.utils.vercmp_string_op, '0', '0', '$') + + +class Path(unittest.TestCase): + def test_unsafe_delete_path(self): + checkitems = [('/', True), + ('//', True), + ('///', True), + (os.getcwd().count(os.sep) * ('..' + os.sep), True), + (os.environ.get('HOME', '/home/test'), True), + ('/home/someone', True), + ('/home/other/', True), + ('/home/other/subdir', False), + ('', False)] + for arg1, correctresult in checkitems: + result = bb.utils._check_unsafe_delete_path(arg1) + self.assertEqual(result, correctresult, '_check_unsafe_delete_path("%s") != %s' % (arg1, correctresult)) + + +class EditMetadataFile(unittest.TestCase): + _origfile = """ +# A comment +HELLO = "oldvalue" + +THIS = "that" + +# Another comment +NOCHANGE = "samevalue" +OTHER = 'anothervalue' + +MULTILINE = "a1 \\ + a2 \\ + a3" + +MULTILINE2 := " \\ + b1 \\ + b2 \\ + b3 \\ + " + + +MULTILINE3 = " \\ + c1 \\ + c2 \\ + c3 \\ +" + +do_functionname() { + command1 ${VAL1} ${VAL2} + command2 ${VAL3} ${VAL4} +} +""" + def _testeditfile(self, varvalues, compareto, dummyvars=None): + if dummyvars is None: + dummyvars = [] + with tempfile.NamedTemporaryFile('w', delete=False) as tf: + tf.write(self._origfile) + tf.close() + try: + varcalls = [] + def handle_file(varname, origvalue, op, newlines): + self.assertIn(varname, varvalues, 'Callback called for variable %s not in the list!' % varname) + self.assertNotIn(varname, dummyvars, 'Callback called for variable %s in dummy list!' % varname) + varcalls.append(varname) + return varvalues[varname] + + bb.utils.edit_metadata_file(tf.name, varvalues.keys(), handle_file) + with open(tf.name) as f: + modfile = f.readlines() + # Ensure the output matches the expected output + self.assertEqual(compareto.splitlines(True), modfile) + # Ensure the callback function was called for every variable we asked for + # (plus allow testing behaviour when a requested variable is not present) + self.assertEqual(sorted(varvalues.keys()), sorted(varcalls + dummyvars)) + finally: + os.remove(tf.name) + + + def test_edit_metadata_file_nochange(self): + # Test file doesn't get modified with nothing to do + self._testeditfile({}, self._origfile) + # Test file doesn't get modified with only dummy variables + self._testeditfile({'DUMMY1': ('should_not_set', None, 0, True), + 'DUMMY2': ('should_not_set_again', None, 0, True)}, self._origfile, dummyvars=['DUMMY1', 'DUMMY2']) + # Test file doesn't get modified with some the same values + self._testeditfile({'THIS': ('that', None, 0, True), + 'OTHER': ('anothervalue', None, 0, True), + 'MULTILINE3': (' c1 c2 c3 ', None, 4, False)}, self._origfile) + + def test_edit_metadata_file_1(self): + + newfile1 = """ +# A comment +HELLO = "newvalue" + +THIS = "that" + +# Another comment +NOCHANGE = "samevalue" +OTHER = 'anothervalue' + +MULTILINE = "a1 \\ + a2 \\ + a3" + +MULTILINE2 := " \\ + b1 \\ + b2 \\ + b3 \\ + " + + +MULTILINE3 = " \\ + c1 \\ + c2 \\ + c3 \\ +" + +do_functionname() { + command1 ${VAL1} ${VAL2} + command2 ${VAL3} ${VAL4} +} +""" + self._testeditfile({'HELLO': ('newvalue', None, 4, True)}, newfile1) + + + def test_edit_metadata_file_2(self): + + newfile2 = """ +# A comment +HELLO = "oldvalue" + +THIS = "that" + +# Another comment +NOCHANGE = "samevalue" +OTHER = 'anothervalue' + +MULTILINE = " \\ + d1 \\ + d2 \\ + d3 \\ + " + +MULTILINE2 := " \\ + b1 \\ + b2 \\ + b3 \\ + " + + +MULTILINE3 = "nowsingle" + +do_functionname() { + command1 ${VAL1} ${VAL2} + command2 ${VAL3} ${VAL4} +} +""" + self._testeditfile({'MULTILINE': (['d1','d2','d3'], None, 4, False), + 'MULTILINE3': ('nowsingle', None, 4, True), + 'NOTPRESENT': (['a', 'b'], None, 4, False)}, newfile2, dummyvars=['NOTPRESENT']) + + + def test_edit_metadata_file_3(self): + + newfile3 = """ +# A comment +HELLO = "oldvalue" + +# Another comment +NOCHANGE = "samevalue" +OTHER = "yetanothervalue" + +MULTILINE = "e1 \\ + e2 \\ + e3 \\ + " + +MULTILINE2 := "f1 \\ +\tf2 \\ +\t" + + +MULTILINE3 = " \\ + c1 \\ + c2 \\ + c3 \\ +" + +do_functionname() { + othercommand_one a b c + othercommand_two d e f +} +""" + + self._testeditfile({'do_functionname()': (['othercommand_one a b c', 'othercommand_two d e f'], None, 4, False), + 'MULTILINE2': (['f1', 'f2'], None, '\t', True), + 'MULTILINE': (['e1', 'e2', 'e3'], None, -1, True), + 'THIS': (None, None, 0, False), + 'OTHER': ('yetanothervalue', None, 0, True)}, newfile3) + + + def test_edit_metadata_file_4(self): + + newfile4 = """ +# A comment +HELLO = "oldvalue" + +THIS = "that" + +# Another comment +OTHER = 'anothervalue' + +MULTILINE = "a1 \\ + a2 \\ + a3" + +MULTILINE2 := " \\ + b1 \\ + b2 \\ + b3 \\ + " + + +""" + + self._testeditfile({'NOCHANGE': (None, None, 0, False), + 'MULTILINE3': (None, None, 0, False), + 'THIS': ('that', None, 0, False), + 'do_functionname()': (None, None, 0, False)}, newfile4) + + + def test_edit_metadata(self): + newfile5 = """ +# A comment +HELLO = "hithere" + +# A new comment +THIS += "that" + +# Another comment +NOCHANGE = "samevalue" +OTHER = 'anothervalue' + +MULTILINE = "a1 \\ + a2 \\ + a3" + +MULTILINE2 := " \\ + b1 \\ + b2 \\ + b3 \\ + " + + +MULTILINE3 = " \\ + c1 \\ + c2 \\ + c3 \\ +" + +NEWVAR = "value" + +do_functionname() { + command1 ${VAL1} ${VAL2} + command2 ${VAL3} ${VAL4} +} +""" + + + def handle_var(varname, origvalue, op, newlines): + if varname == 'THIS': + newlines.append('# A new comment\n') + elif varname == 'do_functionname()': + newlines.append('NEWVAR = "value"\n') + newlines.append('\n') + valueitem = varvalues.get(varname, None) + if valueitem: + return valueitem + else: + return (origvalue, op, 0, True) + + varvalues = {'HELLO': ('hithere', None, 0, True), 'THIS': ('that', '+=', 0, True)} + varlist = ['HELLO', 'THIS', 'do_functionname()'] + (updated, newlines) = bb.utils.edit_metadata(self._origfile.splitlines(True), varlist, handle_var) + self.assertTrue(updated, 'List should be updated but isn\'t') + self.assertEqual(newlines, newfile5.splitlines(True)) + + # Make sure the orig value matches what we expect it to be + def test_edit_metadata_origvalue(self): + origfile = """ +MULTILINE = " stuff \\ + morestuff" +""" + expected_value = "stuff morestuff" + global value_in_callback + value_in_callback = "" + + def handle_var(varname, origvalue, op, newlines): + global value_in_callback + value_in_callback = origvalue + return (origvalue, op, -1, False) + + bb.utils.edit_metadata(origfile.splitlines(True), + ['MULTILINE'], + handle_var) + + testvalue = re.sub('\s+', ' ', value_in_callback.strip()) + self.assertEqual(expected_value, testvalue) + +class EditBbLayersConf(unittest.TestCase): + + def _test_bblayers_edit(self, before, after, add, remove, notadded, notremoved): + with tempfile.NamedTemporaryFile('w', delete=False) as tf: + tf.write(before) + tf.close() + try: + actual_notadded, actual_notremoved = bb.utils.edit_bblayers_conf(tf.name, add, remove) + with open(tf.name) as f: + actual_after = f.readlines() + self.assertEqual(after.splitlines(True), actual_after) + self.assertEqual(notadded, actual_notadded) + self.assertEqual(notremoved, actual_notremoved) + finally: + os.remove(tf.name) + + + def test_bblayers_remove(self): + before = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS = " \ + /home/user/path/layer1 \ + /home/user/path/layer2 \ + /home/user/path/subpath/layer3 \ + /home/user/path/layer4 \ + " +""" + after = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS = " \ + /home/user/path/layer1 \ + /home/user/path/subpath/layer3 \ + /home/user/path/layer4 \ + " +""" + self._test_bblayers_edit(before, after, + None, + '/home/user/path/layer2', + [], + []) + + + def test_bblayers_add(self): + before = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS = " \ + /home/user/path/layer1 \ + /home/user/path/layer2 \ + /home/user/path/subpath/layer3 \ + /home/user/path/layer4 \ + " +""" + after = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS = " \ + /home/user/path/layer1 \ + /home/user/path/layer2 \ + /home/user/path/subpath/layer3 \ + /home/user/path/layer4 \ + /other/path/to/layer5 \ + " +""" + self._test_bblayers_edit(before, after, + '/other/path/to/layer5/', + None, + [], + []) + + + def test_bblayers_add_remove(self): + before = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS = " \ + /home/user/path/layer1 \ + /home/user/path/layer2 \ + /home/user/path/subpath/layer3 \ + /home/user/path/layer4 \ + " +""" + after = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS = " \ + /home/user/path/layer1 \ + /home/user/path/layer2 \ + /home/user/path/layer4 \ + /other/path/to/layer5 \ + " +""" + self._test_bblayers_edit(before, after, + ['/other/path/to/layer5', '/home/user/path/layer2/'], '/home/user/path/subpath/layer3/', + ['/home/user/path/layer2'], + []) + + + def test_bblayers_add_remove_home(self): + before = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS = " \ + ~/path/layer1 \ + ~/path/layer2 \ + ~/otherpath/layer3 \ + ~/path/layer4 \ + " +""" + after = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS = " \ + ~/path/layer2 \ + ~/path/layer4 \ + ~/path2/layer5 \ + " +""" + self._test_bblayers_edit(before, after, + [os.environ['HOME'] + '/path/layer4', '~/path2/layer5'], + [os.environ['HOME'] + '/otherpath/layer3', '~/path/layer1', '~/path/notinlist'], + [os.environ['HOME'] + '/path/layer4'], + ['~/path/notinlist']) + + + def test_bblayers_add_remove_plusequals(self): + before = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS += " \ + /home/user/path/layer1 \ + /home/user/path/layer2 \ + " +""" + after = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS += " \ + /home/user/path/layer2 \ + /home/user/path/layer3 \ + " +""" + self._test_bblayers_edit(before, after, + '/home/user/path/layer3', + '/home/user/path/layer1', + [], + []) + + + def test_bblayers_add_remove_plusequals2(self): + before = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS += " \ + /home/user/path/layer1 \ + /home/user/path/layer2 \ + /home/user/path/layer3 \ + " +BBLAYERS += "/home/user/path/layer4" +BBLAYERS += "/home/user/path/layer5" +""" + after = r""" +# A comment + +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS += " \ + /home/user/path/layer2 \ + /home/user/path/layer3 \ + " +BBLAYERS += "/home/user/path/layer5" +BBLAYERS += "/home/user/otherpath/layer6" +""" + self._test_bblayers_edit(before, after, + ['/home/user/otherpath/layer6', '/home/user/path/layer3'], ['/home/user/path/layer1', '/home/user/path/layer4', '/home/user/path/layer7'], + ['/home/user/path/layer3'], + ['/home/user/path/layer7']) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py b/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py new file mode 100644 index 000000000..7aa653f1a --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py @@ -0,0 +1,105 @@ +# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities +# +# Copyright (C) 2012 Intel Corporation +# Copyright (C) 2011 Mentor Graphics Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import logging +import warnings +import os +import sys + +import bb.cache +import bb.cooker +import bb.providers +import bb.utils +from bb.cooker import state, BBCooker, CookerFeatures +from bb.cookerdata import CookerConfiguration, ConfigParameters +import bb.fetch2 + +class Tinfoil: + def __init__(self, output=sys.stdout, tracking=False): + # Needed to avoid deprecation warnings with python 2.6 + warnings.filterwarnings("ignore", category=DeprecationWarning) + + # Set up logging + self.logger = logging.getLogger('BitBake') + self._log_hdlr = logging.StreamHandler(output) + bb.msg.addDefaultlogFilter(self._log_hdlr) + format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") + if output.isatty(): + format.enable_color() + self._log_hdlr.setFormatter(format) + self.logger.addHandler(self._log_hdlr) + + self.config = CookerConfiguration() + configparams = TinfoilConfigParameters(parse_only=True) + self.config.setConfigParameters(configparams) + self.config.setServerRegIdleCallback(self.register_idle_function) + features = [] + if tracking: + features.append(CookerFeatures.BASEDATASTORE_TRACKING) + self.cooker = BBCooker(self.config, features) + self.config_data = self.cooker.data + bb.providers.logger.setLevel(logging.ERROR) + self.cooker_data = None + + def register_idle_function(self, function, data): + pass + + def parseRecipes(self): + sys.stderr.write("Parsing recipes..") + self.logger.setLevel(logging.WARNING) + + try: + while self.cooker.state in (state.initial, state.parsing): + self.cooker.updateCache() + except KeyboardInterrupt: + self.cooker.shutdown() + self.cooker.updateCache() + sys.exit(2) + + self.logger.setLevel(logging.INFO) + sys.stderr.write("done.\n") + + self.cooker_data = self.cooker.recipecache + + def prepare(self, config_only = False): + if not self.cooker_data: + if config_only: + self.cooker.parseConfiguration() + self.cooker_data = self.cooker.recipecache + else: + self.parseRecipes() + + def shutdown(self): + self.cooker.shutdown(force=True) + self.cooker.post_serve() + self.cooker.unlockBitbake() + self.logger.removeHandler(self._log_hdlr) + +class TinfoilConfigParameters(ConfigParameters): + + def __init__(self, **options): + self.initial_options = options + super(TinfoilConfigParameters, self).__init__() + + def parseCommandLine(self, argv=sys.argv): + class DummyOptions: + def __init__(self, initial_options): + for key, val in initial_options.items(): + setattr(self, key, val) + + return DummyOptions(self.initial_options), None diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/__init__.py new file mode 100644 index 000000000..a4805ed02 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/__init__.py @@ -0,0 +1,17 @@ +# +# BitBake UI Implementation +# +# Copyright (C) 2006-2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py new file mode 100644 index 000000000..93979054d --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py @@ -0,0 +1,1521 @@ +# +# BitBake ToasterUI Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import bb +import re +import os + +os.environ["DJANGO_SETTINGS_MODULE"] = "toaster.toastermain.settings" + + +import django +from django.utils import timezone + + +def _configure_toaster(): + """ Add toaster to sys path for importing modules + """ + sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'toaster')) +_configure_toaster() + +django.setup() + +from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText +from orm.models import Target_Image_File, BuildArtifact +from orm.models import Variable, VariableHistory +from orm.models import Package, Package_File, Target_Installed_Package, Target_File +from orm.models import Task_Dependency, Package_Dependency +from orm.models import Recipe_Dependency, Provides +from orm.models import Project, CustomImagePackage, CustomImageRecipe + +from bldcontrol.models import BuildEnvironment, BuildRequest + +from bb.msg import BBLogFormatter as formatter +from django.db import models +from pprint import pformat +import logging +from datetime import datetime, timedelta + +from django.db import transaction, connection + +# pylint: disable=invalid-name +# the logger name is standard throughout BitBake +logger = logging.getLogger("ToasterLogger") + + +class NotExisting(Exception): + pass + +class ORMWrapper(object): + """ This class creates the dictionaries needed to store information in the database + following the format defined by the Django models. It is also used to save this + information in the database. + """ + + def __init__(self): + self.layer_version_objects = [] + self.layer_version_built = [] + self.task_objects = {} + self.recipe_objects = {} + + @staticmethod + def _build_key(**kwargs): + key = "0" + for k in sorted(kwargs.keys()): + if isinstance(kwargs[k], models.Model): + key += "-%d" % kwargs[k].id + else: + key += "-%s" % str(kwargs[k]) + return key + + + def _cached_get_or_create(self, clazz, **kwargs): + """ This is a memory-cached get_or_create. We assume that the objects will not be created in the + database through any other means. + """ + + assert issubclass(clazz, models.Model), "_cached_get_or_create needs to get the class as first argument" + + key = ORMWrapper._build_key(**kwargs) + dictname = "objects_%s" % clazz.__name__ + if not dictname in vars(self).keys(): + vars(self)[dictname] = {} + + created = False + if not key in vars(self)[dictname].keys(): + vars(self)[dictname][key], created = \ + clazz.objects.get_or_create(**kwargs) + + return (vars(self)[dictname][key], created) + + + def _cached_get(self, clazz, **kwargs): + """ This is a memory-cached get. We assume that the objects will not change in the database between gets. + """ + assert issubclass(clazz, models.Model), "_cached_get needs to get the class as first argument" + + key = ORMWrapper._build_key(**kwargs) + dictname = "objects_%s" % clazz.__name__ + + if not dictname in vars(self).keys(): + vars(self)[dictname] = {} + + if not key in vars(self)[dictname].keys(): + vars(self)[dictname][key] = clazz.objects.get(**kwargs) + + return vars(self)[dictname][key] + + def _timestamp_to_datetime(self, secs): + """ + Convert timestamp in seconds to Python datetime + """ + return datetime(1970, 1, 1) + timedelta(seconds=secs) + + # pylint: disable=no-self-use + # we disable detection of no self use in functions because the methods actually work on the object + # even if they don't touch self anywhere + + # pylint: disable=bad-continuation + # we do not follow the python conventions for continuation indentation due to long lines here + + def create_build_object(self, build_info, brbe, project_id): + assert 'machine' in build_info + assert 'distro' in build_info + assert 'distro_version' in build_info + assert 'started_on' in build_info + assert 'cooker_log_path' in build_info + assert 'build_name' in build_info + assert 'bitbake_version' in build_info + + prj = None + buildrequest = None + if brbe is not None: # this build was triggered by a request from a user + logger.debug(1, "buildinfohelper: brbe is %s" % brbe) + br, _ = brbe.split(":") + buildrequest = BuildRequest.objects.get(pk = br) + prj = buildrequest.project + + elif project_id is not None: # this build was triggered by an external system for a specific project + logger.debug(1, "buildinfohelper: project is %s" % prj) + prj = Project.objects.get(pk = project_id) + + else: # this build was triggered by a legacy system, or command line interactive mode + prj = Project.objects.get_or_create_default_project() + logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) + + + if buildrequest is not None: + build = buildrequest.build + logger.info("Updating existing build, with %s", build_info) + build.project = prj + build.machine=build_info['machine'] + build.distro=build_info['distro'] + build.distro_version=build_info['distro_version'] + build.cooker_log_path=build_info['cooker_log_path'] + build.build_name=build_info['build_name'] + build.bitbake_version=build_info['bitbake_version'] + build.save() + + else: + build = Build.objects.create( + project = prj, + machine=build_info['machine'], + distro=build_info['distro'], + distro_version=build_info['distro_version'], + started_on=build_info['started_on'], + completed_on=build_info['started_on'], + cooker_log_path=build_info['cooker_log_path'], + build_name=build_info['build_name'], + bitbake_version=build_info['bitbake_version']) + + logger.debug(1, "buildinfohelper: build is created %s" % build) + + if buildrequest is not None: + buildrequest.build = build + buildrequest.save() + + return build + + @staticmethod + def get_or_create_targets(target_info): + result = [] + for target in target_info['targets']: + task = '' + if ':' in target: + target, task = target.split(':', 1) + if task.startswith('do_'): + task = task[3:] + if task == 'build': + task = '' + obj, created = Target.objects.get_or_create(build=target_info['build'], + target=target) + if created: + obj.is_image = False + if task: + obj.task = task + obj.save() + result.append(obj) + return result + + def update_build_object(self, build, errors, warnings, taskfailures): + assert isinstance(build,Build) + assert isinstance(errors, int) + assert isinstance(warnings, int) + + if build.outcome == Build.CANCELLED: + return + try: + if build.buildrequest.state == BuildRequest.REQ_CANCELLING: + return + except AttributeError: + # We may not have a buildrequest if this is a command line build + pass + + outcome = Build.SUCCEEDED + if errors or taskfailures: + outcome = Build.FAILED + + build.completed_on = timezone.now() + build.outcome = outcome + build.save() + + def update_target_set_license_manifest(self, target, license_manifest_path): + target.license_manifest_path = license_manifest_path + target.save() + + def update_task_object(self, build, task_name, recipe_name, task_stats): + """ + Find the task for build which matches the recipe and task name + to be stored + """ + task_to_update = Task.objects.get( + build = build, + task_name = task_name, + recipe__name = recipe_name + ) + + if 'started' in task_stats and 'ended' in task_stats: + task_to_update.started = self._timestamp_to_datetime(task_stats['started']) + task_to_update.ended = self._timestamp_to_datetime(task_stats['ended']) + task_to_update.elapsed_time = (task_stats['ended'] - task_stats['started']) + task_to_update.cpu_time_user = task_stats.get('cpu_time_user') + task_to_update.cpu_time_system = task_stats.get('cpu_time_system') + if 'disk_io_read' in task_stats and 'disk_io_write' in task_stats: + task_to_update.disk_io_read = task_stats['disk_io_read'] + task_to_update.disk_io_write = task_stats['disk_io_write'] + task_to_update.disk_io = task_stats['disk_io_read'] + task_stats['disk_io_write'] + + task_to_update.save() + + def get_update_task_object(self, task_information, must_exist = False): + assert 'build' in task_information + assert 'recipe' in task_information + assert 'task_name' in task_information + + # we use must_exist info for database look-up optimization + task_object, created = self._cached_get_or_create(Task, + build=task_information['build'], + recipe=task_information['recipe'], + task_name=task_information['task_name'] + ) + if created and must_exist: + task_information['debug'] = "build id %d, recipe id %d" % (task_information['build'].pk, task_information['recipe'].pk) + raise NotExisting("Task object created when expected to exist", task_information) + + object_changed = False + for v in vars(task_object): + if v in task_information.keys(): + if vars(task_object)[v] != task_information[v]: + vars(task_object)[v] = task_information[v] + object_changed = True + + # update setscene-related information if the task has a setscene + if task_object.outcome == Task.OUTCOME_COVERED and 1 == task_object.get_related_setscene().count(): + task_object.outcome = Task.OUTCOME_CACHED + object_changed = True + + outcome_task_setscene = Task.objects.get(task_executed=True, build = task_object.build, + recipe = task_object.recipe, task_name=task_object.task_name+"_setscene").outcome + if outcome_task_setscene == Task.OUTCOME_SUCCESS: + task_object.sstate_result = Task.SSTATE_RESTORED + object_changed = True + elif outcome_task_setscene == Task.OUTCOME_FAILED: + task_object.sstate_result = Task.SSTATE_FAILED + object_changed = True + + if object_changed: + task_object.save() + return task_object + + + def get_update_recipe_object(self, recipe_information, must_exist = False): + assert 'layer_version' in recipe_information + assert 'file_path' in recipe_information + assert 'pathflags' in recipe_information + + assert not recipe_information['file_path'].startswith("/") # we should have layer-relative paths at all times + + + def update_recipe_obj(recipe_object): + object_changed = False + for v in vars(recipe_object): + if v in recipe_information.keys(): + object_changed = True + vars(recipe_object)[v] = recipe_information[v] + + if object_changed: + recipe_object.save() + + recipe, created = self._cached_get_or_create(Recipe, layer_version=recipe_information['layer_version'], + file_path=recipe_information['file_path'], pathflags = recipe_information['pathflags']) + + update_recipe_obj(recipe) + + built_recipe = None + # Create a copy of the recipe for historical puposes and update it + for built_layer in self.layer_version_built: + if built_layer.layer == recipe_information['layer_version'].layer: + built_recipe, c = self._cached_get_or_create(Recipe, + layer_version=built_layer, + file_path=recipe_information['file_path'], + pathflags = recipe_information['pathflags']) + update_recipe_obj(built_recipe) + break + + + # If we're in analysis mode or if this is a custom recipe + # then we are wholly responsible for the data + # and therefore we return the 'real' recipe rather than the build + # history copy of the recipe. + if recipe_information['layer_version'].build is not None and \ + recipe_information['layer_version'].build.project == \ + Project.objects.get_or_create_default_project(): + return recipe + + if built_recipe is None: + return recipe + + return built_recipe + + def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information): + if isinstance(layer_obj, Layer_Version): + # Special case the toaster-custom-images layer which is created + # on the fly so don't update the values which may cause the layer + # to be duplicated on a future get_or_create + if layer_obj.layer.name == CustomImageRecipe.LAYER_NAME: + return layer_obj + # We already found our layer version for this build so just + # update it with the new build information + logger.debug("We found our layer from toaster") + layer_obj.local_path = layer_version_information['local_path'] + layer_obj.save() + self.layer_version_objects.append(layer_obj) + + # create a new copy of this layer version as a snapshot for + # historical purposes + layer_copy, c = Layer_Version.objects.get_or_create( + build=build_obj, + layer=layer_obj.layer, + up_branch=layer_obj.up_branch, + branch=layer_version_information['branch'], + commit=layer_version_information['commit'], + local_path=layer_version_information['local_path'], + ) + + logger.info("created new historical layer version %d", + layer_copy.pk) + + self.layer_version_built.append(layer_copy) + + return layer_obj + + assert isinstance(build_obj, Build) + assert isinstance(layer_obj, Layer) + assert 'branch' in layer_version_information + assert 'commit' in layer_version_information + assert 'priority' in layer_version_information + assert 'local_path' in layer_version_information + + # If we're doing a command line build then associate this new layer with the + # project to avoid it 'contaminating' toaster data + project = None + if build_obj.project == Project.objects.get_or_create_default_project(): + project = build_obj.project + + layer_version_object, _ = Layer_Version.objects.get_or_create( + build = build_obj, + layer = layer_obj, + branch = layer_version_information['branch'], + commit = layer_version_information['commit'], + priority = layer_version_information['priority'], + local_path = layer_version_information['local_path'], + project=project) + + self.layer_version_objects.append(layer_version_object) + + return layer_version_object + + def get_update_layer_object(self, layer_information, brbe): + assert 'name' in layer_information + assert 'layer_index_url' in layer_information + + if brbe is None: + layer_object, _ = Layer.objects.get_or_create( + name=layer_information['name'], + layer_index_url=layer_information['layer_index_url']) + return layer_object + else: + # we are under managed mode; we must match the layer used in the Project Layer + br_id, be_id = brbe.split(":") + + # find layer by checkout path; + from bldcontrol import bbcontroller + bc = bbcontroller.getBuildEnvironmentController(pk = be_id) + + # we might have a race condition here, as the project layers may change between the build trigger and the actual build execution + # but we can only match on the layer name, so the worst thing can happen is a mis-identification of the layer, not a total failure + + # note that this is different + buildrequest = BuildRequest.objects.get(pk = br_id) + for brl in buildrequest.brlayer_set.all(): + localdirname = os.path.join(bc.getGitCloneDirectory(brl.giturl, brl.commit), brl.dirpath) + # we get a relative path, unless running in HEAD mode where the path is absolute + if not localdirname.startswith("/"): + localdirname = os.path.join(bc.be.sourcedir, localdirname) + #logger.debug(1, "Localdirname %s lcal_path %s" % (localdirname, layer_information['local_path'])) + if localdirname.startswith(layer_information['local_path']): + # If the build request came from toaster this field + # should contain the information from the layer_version + # That created this build request. + if brl.layer_version: + return brl.layer_version + + # we matched the BRLayer, but we need the layer_version that generated this BR; reverse of the Project.schedule_build() + #logger.debug(1, "Matched %s to BRlayer %s" % (pformat(layer_information["local_path"]), localdirname)) + + for pl in buildrequest.project.projectlayer_set.filter(layercommit__layer__name = brl.name): + if pl.layercommit.layer.vcs_url == brl.giturl : + layer = pl.layercommit.layer + layer.save() + return layer + + raise NotExisting("Unidentified layer %s" % pformat(layer_information)) + + + def save_target_file_information(self, build_obj, target_obj, filedata): + assert isinstance(build_obj, Build) + assert isinstance(target_obj, Target) + dirs = filedata['dirs'] + files = filedata['files'] + syms = filedata['syms'] + + # always create the root directory as a special case; + # note that this is never displayed, so the owner, group, + # size, permission are irrelevant + tf_obj = Target_File.objects.create(target = target_obj, + path = '/', + size = 0, + owner = '', + group = '', + permission = '', + inodetype = Target_File.ITYPE_DIRECTORY) + tf_obj.save() + + # insert directories, ordered by name depth + for d in sorted(dirs, key=lambda x:len(x[-1].split("/"))): + (user, group, size) = d[1:4] + permission = d[0][1:] + path = d[4].lstrip(".") + + # we already created the root directory, so ignore any + # entry for it + if len(path) == 0: + continue + + parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) + if len(parent_path) == 0: + parent_path = "/" + parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) + tf_obj = Target_File.objects.create( + target = target_obj, + path = unicode(path, 'utf-8'), + size = size, + inodetype = Target_File.ITYPE_DIRECTORY, + permission = permission, + owner = user, + group = group, + directory = parent_obj) + + + # we insert files + for d in files: + (user, group, size) = d[1:4] + permission = d[0][1:] + path = d[4].lstrip(".") + parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) + inodetype = Target_File.ITYPE_REGULAR + if d[0].startswith('b'): + inodetype = Target_File.ITYPE_BLOCK + if d[0].startswith('c'): + inodetype = Target_File.ITYPE_CHARACTER + if d[0].startswith('p'): + inodetype = Target_File.ITYPE_FIFO + + tf_obj = Target_File.objects.create( + target = target_obj, + path = unicode(path, 'utf-8'), + size = size, + inodetype = inodetype, + permission = permission, + owner = user, + group = group) + parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) + tf_obj.directory = parent_obj + tf_obj.save() + + # we insert symlinks + for d in syms: + (user, group, size) = d[1:4] + permission = d[0][1:] + path = d[4].lstrip(".") + filetarget_path = d[6] + + parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) + if not filetarget_path.startswith("/"): + # we have a relative path, get a normalized absolute one + filetarget_path = parent_path + "/" + filetarget_path + fcp = filetarget_path.split("/") + fcpl = [] + for i in fcp: + if i == "..": + fcpl.pop() + else: + fcpl.append(i) + filetarget_path = "/".join(fcpl) + + try: + filetarget_obj = Target_File.objects.get( + target = target_obj, + path = unicode(filetarget_path, 'utf-8')) + except Target_File.DoesNotExist: + # we might have an invalid link; no way to detect this. just set it to None + filetarget_obj = None + + parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) + + tf_obj = Target_File.objects.create( + target = target_obj, + path = unicode(path, 'utf-8'), + size = size, + inodetype = Target_File.ITYPE_SYMLINK, + permission = permission, + owner = user, + group = group, + directory = parent_obj, + sym_target = filetarget_obj) + + + def save_target_package_information(self, build_obj, target_obj, packagedict, pkgpnmap, recipes, built_package=False): + assert isinstance(build_obj, Build) + assert isinstance(target_obj, Target) + + errormsg = "" + for p in packagedict: + # Search name swtiches round the installed name vs package name + # by default installed name == package name + searchname = p + if p not in pkgpnmap: + logger.warning("Image packages list contains %p, but is" + " missing from all packages list where the" + " metadata comes from. Skipping...", p) + continue + + if 'OPKGN' in pkgpnmap[p].keys(): + searchname = pkgpnmap[p]['OPKGN'] + + built_recipe = recipes[pkgpnmap[p]['PN']] + + if built_package: + packagedict[p]['object'], created = Package.objects.get_or_create( build = build_obj, name = searchname ) + recipe = built_recipe + else: + packagedict[p]['object'], created = \ + CustomImagePackage.objects.get_or_create(name=searchname) + # Clear the Package_Dependency objects as we're going to update + # the CustomImagePackage with the latest dependency information + packagedict[p]['object'].package_dependencies_target.all().delete() + packagedict[p]['object'].package_dependencies_source.all().delete() + try: + recipe = self._cached_get( + Recipe, + name=built_recipe.name, + layer_version__build=None, + layer_version__up_branch= + built_recipe.layer_version.up_branch, + file_path=built_recipe.file_path, + version=built_recipe.version + ) + except (Recipe.DoesNotExist, + Recipe.MultipleObjectsReturned) as e: + logger.info("We did not find one recipe for the" + "configuration data package %s %s" % (p, e)) + continue + + if created or packagedict[p]['object'].size == -1: # save the data anyway we can, not just if it was not created here; bug [YOCTO #6887] + # fill in everything we can from the runtime-reverse package data + try: + packagedict[p]['object'].recipe = recipe + packagedict[p]['object'].version = pkgpnmap[p]['PV'] + packagedict[p]['object'].installed_name = p + packagedict[p]['object'].revision = pkgpnmap[p]['PR'] + packagedict[p]['object'].license = pkgpnmap[p]['LICENSE'] + packagedict[p]['object'].section = pkgpnmap[p]['SECTION'] + packagedict[p]['object'].summary = pkgpnmap[p]['SUMMARY'] + packagedict[p]['object'].description = pkgpnmap[p]['DESCRIPTION'] + packagedict[p]['object'].size = int(pkgpnmap[p]['PKGSIZE']) + + # no files recorded for this package, so save files info + packagefile_objects = [] + for targetpath in pkgpnmap[p]['FILES_INFO']: + targetfilesize = pkgpnmap[p]['FILES_INFO'][targetpath] + packagefile_objects.append(Package_File( package = packagedict[p]['object'], + path = targetpath, + size = targetfilesize)) + if len(packagefile_objects): + Package_File.objects.bulk_create(packagefile_objects) + except KeyError as e: + errormsg += " stpi: Key error, package %s key %s \n" % ( p, e ) + + # save disk installed size + packagedict[p]['object'].installed_size = packagedict[p]['size'] + packagedict[p]['object'].save() + + if built_package: + Target_Installed_Package.objects.create(target = target_obj, package = packagedict[p]['object']) + + packagedeps_objs = [] + for p in packagedict: + for (px,deptype) in packagedict[p]['depends']: + if deptype == 'depends': + tdeptype = Package_Dependency.TYPE_TRDEPENDS + elif deptype == 'recommends': + tdeptype = Package_Dependency.TYPE_TRECOMMENDS + + try: + packagedeps_objs.append(Package_Dependency( + package = packagedict[p]['object'], + depends_on = packagedict[px]['object'], + dep_type = tdeptype, + target = target_obj)) + except KeyError as e: + logger.warn("Could not add dependency to the package %s " + "because %s is an unknown package", p, px) + + if len(packagedeps_objs) > 0: + Package_Dependency.objects.bulk_create(packagedeps_objs) + else: + logger.info("No package dependencies created") + + if len(errormsg) > 0: + logger.warn("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) + + def save_target_image_file_information(self, target_obj, file_name, file_size): + Target_Image_File.objects.create( target = target_obj, + file_name = file_name, + file_size = file_size) + + def save_artifact_information(self, build_obj, file_name, file_size): + # we skip the image files from other builds + if Target_Image_File.objects.filter(file_name = file_name).count() > 0: + return + + # do not update artifacts found in other builds + if BuildArtifact.objects.filter(file_name = file_name).count() > 0: + return + + BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size) + + def create_logmessage(self, log_information): + assert 'build' in log_information + assert 'level' in log_information + assert 'message' in log_information + + log_object = LogMessage.objects.create( + build = log_information['build'], + level = log_information['level'], + message = log_information['message']) + + for v in vars(log_object): + if v in log_information.keys(): + vars(log_object)[v] = log_information[v] + + return log_object.save() + + + def save_build_package_information(self, build_obj, package_info, recipes, + built_package): + # assert isinstance(build_obj, Build) + + # create and save the object + pname = package_info['PKG'] + built_recipe = recipes[package_info['PN']] + if 'OPKGN' in package_info.keys(): + pname = package_info['OPKGN'] + + if built_package: + bp_object, _ = Package.objects.get_or_create( build = build_obj, + name = pname ) + recipe = built_recipe + else: + bp_object, created = \ + CustomImagePackage.objects.get_or_create(name=pname) + try: + recipe = self._cached_get(Recipe, + name=built_recipe.name, + layer_version__build=None, + file_path=built_recipe.file_path, + version=built_recipe.version) + + except (Recipe.DoesNotExist, Recipe.MultipleObjectsReturned): + logger.debug("We did not find one recipe for the configuration" + "data package %s" % pname) + return + + bp_object.installed_name = package_info['PKG'] + bp_object.recipe = recipe + bp_object.version = package_info['PKGV'] + bp_object.revision = package_info['PKGR'] + bp_object.summary = package_info['SUMMARY'] + bp_object.description = package_info['DESCRIPTION'] + bp_object.size = int(package_info['PKGSIZE']) + bp_object.section = package_info['SECTION'] + bp_object.license = package_info['LICENSE'] + bp_object.save() + + # save any attached file information + packagefile_objects = [] + for path in package_info['FILES_INFO']: + packagefile_objects.append(Package_File( package = bp_object, + path = path, + size = package_info['FILES_INFO'][path] )) + if len(packagefile_objects): + Package_File.objects.bulk_create(packagefile_objects) + + def _po_byname(p): + if built_package: + pkg, created = Package.objects.get_or_create(build=build_obj, + name=p) + else: + pkg, created = CustomImagePackage.objects.get_or_create(name=p) + + if created: + pkg.size = -1 + pkg.save() + return pkg + + packagedeps_objs = [] + # save soft dependency information + if 'RDEPENDS' in package_info and package_info['RDEPENDS']: + for p in bb.utils.explode_deps(package_info['RDEPENDS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RDEPENDS)) + if 'RPROVIDES' in package_info and package_info['RPROVIDES']: + for p in bb.utils.explode_deps(package_info['RPROVIDES']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RPROVIDES)) + if 'RRECOMMENDS' in package_info and package_info['RRECOMMENDS']: + for p in bb.utils.explode_deps(package_info['RRECOMMENDS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RRECOMMENDS)) + if 'RSUGGESTS' in package_info and package_info['RSUGGESTS']: + for p in bb.utils.explode_deps(package_info['RSUGGESTS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RSUGGESTS)) + if 'RREPLACES' in package_info and package_info['RREPLACES']: + for p in bb.utils.explode_deps(package_info['RREPLACES']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RREPLACES)) + if 'RCONFLICTS' in package_info and package_info['RCONFLICTS']: + for p in bb.utils.explode_deps(package_info['RCONFLICTS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS)) + + if len(packagedeps_objs) > 0: + Package_Dependency.objects.bulk_create(packagedeps_objs) + + return bp_object + + def save_build_variables(self, build_obj, vardump): + assert isinstance(build_obj, Build) + + for k in vardump: + desc = vardump[k]['doc'] + if desc is None: + var_words = [word for word in k.split('_')] + root_var = "_".join([word for word in var_words if word.isupper()]) + if root_var and root_var != k and root_var in vardump: + desc = vardump[root_var]['doc'] + if desc is None: + desc = '' + if len(desc): + HelpText.objects.get_or_create(build=build_obj, + area=HelpText.VARIABLE, + key=k, text=desc) + if not bool(vardump[k]['func']): + value = vardump[k]['v'] + if value is None: + value = '' + variable_obj = Variable.objects.create( build = build_obj, + variable_name = k, + variable_value = value, + description = desc) + + varhist_objects = [] + for vh in vardump[k]['history']: + if not 'documentation.conf' in vh['file']: + varhist_objects.append(VariableHistory( variable = variable_obj, + file_name = vh['file'], + line_number = vh['line'], + operation = vh['op'])) + if len(varhist_objects): + VariableHistory.objects.bulk_create(varhist_objects) + + +class MockEvent(object): + """ This object is used to create event, for which normal event-processing methods can + be used, out of data that is not coming via an actual event + """ + def __init__(self): + self.msg = None + self.levelno = None + self.taskname = None + self.taskhash = None + self.pathname = None + self.lineno = None + + +class BuildInfoHelper(object): + """ This class gathers the build information from the server and sends it + towards the ORM wrapper for storing in the database + It is instantiated once per build + Keeps in memory all data that needs matching before writing it to the database + """ + + # pylint: disable=protected-access + # the code will look into the protected variables of the event; no easy way around this + # pylint: disable=bad-continuation + # we do not follow the python conventions for continuation indentation due to long lines here + + def __init__(self, server, has_build_history = False, brbe = None): + self.internal_state = {} + self.internal_state['taskdata'] = {} + self.internal_state['targets'] = [] + self.task_order = 0 + self.autocommit_step = 1 + self.server = server + # we use manual transactions if the database doesn't autocommit on us + if not connection.features.autocommits_when_autocommit_is_off: + transaction.set_autocommit(False) + self.orm_wrapper = ORMWrapper() + self.has_build_history = has_build_history + self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0] + + # this is set for Toaster-triggered builds by localhostbecontroller + # via toasterui + self.brbe = brbe + + self.project = None + + logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self)) + + + ################### + ## methods to convert event/external info into objects that the ORM layer uses + + + def _get_build_information(self, build_log_path): + build_info = {} + build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0] + build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0] + build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] + build_info['started_on'] = timezone.now() + build_info['completed_on'] = timezone.now() + build_info['cooker_log_path'] = build_log_path + build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] + build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] + build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0] + return build_info + + def _get_task_information(self, event, recipe): + assert 'taskname' in vars(event) + + task_information = {} + task_information['build'] = self.internal_state['build'] + task_information['outcome'] = Task.OUTCOME_NA + task_information['recipe'] = recipe + task_information['task_name'] = event.taskname + try: + # some tasks don't come with a hash. and that's ok + task_information['sstate_checksum'] = event.taskhash + except AttributeError: + pass + return task_information + + def _get_layer_version_for_path(self, path): + assert path.startswith("/") + assert 'build' in self.internal_state + + def _slkey_interactive(layer_version): + assert isinstance(layer_version, Layer_Version) + return len(layer_version.local_path) + + # Heuristics: we always match recipe to the deepest layer path in the discovered layers + for lvo in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_slkey_interactive): + # we can match to the recipe file path + if path.startswith(lvo.local_path): + return lvo + + #if we get here, we didn't read layers correctly; dump whatever information we have on the error log + logger.warn("Could not match layer version for recipe path %s : %s", path, self.orm_wrapper.layer_version_objects) + + #mockup the new layer + unknown_layer, _ = Layer.objects.get_or_create(name="Unidentified layer", layer_index_url="") + unknown_layer_version_obj, _ = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build']) + + # append it so we don't run into this error again and again + self.orm_wrapper.layer_version_objects.append(unknown_layer_version_obj) + + return unknown_layer_version_obj + + def _get_recipe_information_from_taskfile(self, taskfile): + localfilepath = taskfile.split(":")[-1] + filepath_flags = ":".join(sorted(taskfile.split(":")[:-1])) + layer_version_obj = self._get_layer_version_for_path(localfilepath) + + + + recipe_info = {} + recipe_info['layer_version'] = layer_version_obj + recipe_info['file_path'] = localfilepath + recipe_info['pathflags'] = filepath_flags + + if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path): + recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/") + else: + raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path)) + + return recipe_info + + def _get_path_information(self, task_object): + assert isinstance(task_object, Task) + build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/" + build_stats_path = [] + + for t in self.internal_state['targets']: + buildname = self.internal_state['build'].build_name + pe, pv = task_object.recipe.version.split(":",1) + if len(pe) > 0: + package = task_object.recipe.name + "-" + pe + "_" + pv + else: + package = task_object.recipe.name + "-" + pv + + build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, + buildname=buildname, + package=package)) + + return build_stats_path + + + ################################ + ## external available methods to store information + @staticmethod + def _get_data_from_event(event): + evdata = None + if '_localdata' in vars(event): + evdata = event._localdata + elif 'data' in vars(event): + evdata = event.data + else: + raise Exception("Event with neither _localdata or data properties") + return evdata + + def store_layer_info(self, event): + layerinfos = BuildInfoHelper._get_data_from_event(event) + self.internal_state['lvs'] = {} + for layer in layerinfos: + try: + self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)] = layerinfos[layer]['version'] + self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)]['local_path'] = layerinfos[layer]['local_path'] + except NotExisting as nee: + logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee) + + + def store_started_build(self, event, build_log_path): + assert '_pkgs' in vars(event) + build_information = self._get_build_information(build_log_path) + + # Update brbe and project as they can be changed for every build + self.project = build_information['project'] + + build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) + + self.internal_state['build'] = build_obj + + # save layer version information for this build + if not 'lvs' in self.internal_state: + logger.error("Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass.") + else: + for layer_obj in self.internal_state['lvs']: + self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj]) + + del self.internal_state['lvs'] + + # create target information + target_information = {} + target_information['targets'] = event._pkgs + target_information['build'] = build_obj + + self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) + + # Save build configuration + data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0] + + # convert the paths from absolute to relative to either the build directory or layer checkouts + path_prefixes = [] + + if self.brbe is not None: + _, be_id = self.brbe.split(":") + be = BuildEnvironment.objects.get(pk = be_id) + path_prefixes.append(be.builddir) + + for layer in sorted(self.orm_wrapper.layer_version_objects, key = lambda x:len(x.local_path), reverse=True): + path_prefixes.append(layer.local_path) + + # we strip the prefixes + for k in data: + if not bool(data[k]['func']): + for vh in data[k]['history']: + if not 'documentation.conf' in vh['file']: + abs_file_name = vh['file'] + for pp in path_prefixes: + if abs_file_name.startswith(pp + "/"): + vh['file']=abs_file_name[len(pp + "/"):] + break + + # save the variables + self.orm_wrapper.save_build_variables(build_obj, data) + + return self.brbe + + + def update_target_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + + for t in self.internal_state['targets']: + if t.is_image == True: + output_files = list(evdata.viewkeys()) + for output in output_files: + if t.target in output and 'rootfs' in output and not output.endswith(".manifest"): + self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) + + def update_artifact_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + for artifact_path in evdata.keys(): + self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) + + def update_build_information(self, event, errors, warnings, taskfailures): + if 'build' in self.internal_state: + self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) + + + def store_license_manifest_path(self, event): + deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir'] + image_name = BuildInfoHelper._get_data_from_event(event)['image_name'] + path = deploy_dir + "/licenses/" + image_name + "/license.manifest" + for target in self.internal_state['targets']: + if target.target in image_name: + self.orm_wrapper.update_target_set_license_manifest(target, path) + + + def store_started_task(self, event): + assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)) + assert 'taskfile' in vars(event) + localfilepath = event.taskfile.split(":")[-1] + assert localfilepath.startswith("/") + + identifier = event.taskfile + ":" + event.taskname + + recipe_information = self._get_recipe_information_from_taskfile(event.taskfile) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True) + + task_information = self._get_task_information(event, recipe) + task_information['outcome'] = Task.OUTCOME_NA + + if isinstance(event, bb.runqueue.runQueueTaskSkipped): + assert 'reason' in vars(event) + task_information['task_executed'] = False + if event.reason == "covered": + task_information['outcome'] = Task.OUTCOME_COVERED + if event.reason == "existing": + task_information['outcome'] = Task.OUTCOME_PREBUILT + else: + task_information['task_executed'] = True + if 'noexec' in vars(event) and event.noexec == True: + task_information['task_executed'] = False + task_information['outcome'] = Task.OUTCOME_EMPTY + task_information['script_type'] = Task.CODING_NA + + # do not assign order numbers to scene tasks + if not isinstance(event, bb.runqueue.sceneQueueTaskStarted): + self.task_order += 1 + task_information['order'] = self.task_order + + self.orm_wrapper.get_update_task_object(task_information) + + self.internal_state['taskdata'][identifier] = { + 'outcome': task_information['outcome'], + } + + + def store_tasks_stats(self, event): + task_data = BuildInfoHelper._get_data_from_event(event) + + for (task_file, task_name, task_stats, recipe_name) in task_data: + build = self.internal_state['build'] + self.orm_wrapper.update_task_object(build, task_name, recipe_name, task_stats) + + def update_and_store_task(self, event): + assert 'taskfile' in vars(event) + localfilepath = event.taskfile.split(":")[-1] + assert localfilepath.startswith("/") + + identifier = event.taskfile + ":" + event.taskname + if not identifier in self.internal_state['taskdata']: + if isinstance(event, bb.build.TaskBase): + # we do a bit of guessing + candidates = [x for x in self.internal_state['taskdata'].keys() if x.endswith(identifier)] + if len(candidates) == 1: + identifier = candidates[0] + + assert identifier in self.internal_state['taskdata'] + identifierlist = identifier.split(":") + realtaskfile = ":".join(identifierlist[0:len(identifierlist)-1]) + recipe_information = self._get_recipe_information_from_taskfile(realtaskfile) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True) + task_information = self._get_task_information(event,recipe) + + task_information['outcome'] = self.internal_state['taskdata'][identifier]['outcome'] + + if 'logfile' in vars(event): + task_information['logfile'] = event.logfile + + if '_message' in vars(event): + task_information['message'] = event._message + + if 'taskflags' in vars(event): + # with TaskStarted, we get even more information + if 'python' in event.taskflags.keys() and event.taskflags['python'] == '1': + task_information['script_type'] = Task.CODING_PYTHON + else: + task_information['script_type'] = Task.CODING_SHELL + + if task_information['outcome'] == Task.OUTCOME_NA: + if isinstance(event, (bb.runqueue.runQueueTaskCompleted, bb.runqueue.sceneQueueTaskCompleted)): + task_information['outcome'] = Task.OUTCOME_SUCCESS + del self.internal_state['taskdata'][identifier] + + if isinstance(event, (bb.runqueue.runQueueTaskFailed, bb.runqueue.sceneQueueTaskFailed)): + task_information['outcome'] = Task.OUTCOME_FAILED + del self.internal_state['taskdata'][identifier] + + if not connection.features.autocommits_when_autocommit_is_off: + # we force a sync point here, to get the progress bar to show + if self.autocommit_step % 3 == 0: + transaction.set_autocommit(True) + transaction.set_autocommit(False) + self.autocommit_step += 1 + + self.orm_wrapper.get_update_task_object(task_information, True) # must exist + + + def store_missed_state_tasks(self, event): + for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['missed']: + + # identifier = fn + taskname + "_setscene" + recipe_information = self._get_recipe_information_from_taskfile(fn) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) + mevent = MockEvent() + mevent.taskname = taskname + mevent.taskhash = taskhash + task_information = self._get_task_information(mevent,recipe) + + task_information['start_time'] = timezone.now() + task_information['outcome'] = Task.OUTCOME_NA + task_information['sstate_checksum'] = taskhash + task_information['sstate_result'] = Task.SSTATE_MISS + task_information['path_to_sstate_obj'] = sstatefile + + self.orm_wrapper.get_update_task_object(task_information) + + for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['found']: + + # identifier = fn + taskname + "_setscene" + recipe_information = self._get_recipe_information_from_taskfile(fn) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) + mevent = MockEvent() + mevent.taskname = taskname + mevent.taskhash = taskhash + task_information = self._get_task_information(mevent,recipe) + + task_information['path_to_sstate_obj'] = sstatefile + + self.orm_wrapper.get_update_task_object(task_information) + + + def store_target_package_data(self, event): + # for all image targets + for target in self.internal_state['targets']: + if target.is_image: + pkgdata = BuildInfoHelper._get_data_from_event(event)['pkgdata'] + imgdata = BuildInfoHelper._get_data_from_event(event)['imgdata'].get(target.target, {}) + filedata = BuildInfoHelper._get_data_from_event(event)['filedata'].get(target.target, {}) + + try: + self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes'], built_package=True) + self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata.copy(), pkgdata, self.internal_state['recipes'], built_package=False) + except KeyError as e: + logger.warn("KeyError in save_target_package_information" + "%s ", e) + + try: + self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata) + except KeyError as e: + logger.warn("KeyError in save_target_file_information" + "%s ", e) + + + + + def store_dependency_information(self, event): + assert '_depgraph' in vars(event) + assert 'layer-priorities' in event._depgraph + assert 'pn' in event._depgraph + assert 'tdepends' in event._depgraph + + errormsg = "" + + # save layer version priorities + if 'layer-priorities' in event._depgraph.keys(): + for lv in event._depgraph['layer-priorities']: + (_, path, _, priority) = lv + layer_version_obj = self._get_layer_version_for_path(path[1:]) # paths start with a ^ + assert layer_version_obj is not None + layer_version_obj.priority = priority + layer_version_obj.save() + + # save recipe information + self.internal_state['recipes'] = {} + for pn in event._depgraph['pn']: + + file_name = event._depgraph['pn'][pn]['filename'].split(":")[-1] + pathflags = ":".join(sorted(event._depgraph['pn'][pn]['filename'].split(":")[:-1])) + layer_version_obj = self._get_layer_version_for_path(file_name) + + assert layer_version_obj is not None + + recipe_info = {} + recipe_info['name'] = pn + recipe_info['layer_version'] = layer_version_obj + + if 'version' in event._depgraph['pn'][pn]: + recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":") + + if 'summary' in event._depgraph['pn'][pn]: + recipe_info['summary'] = event._depgraph['pn'][pn]['summary'] + + if 'license' in event._depgraph['pn'][pn]: + recipe_info['license'] = event._depgraph['pn'][pn]['license'] + + if 'description' in event._depgraph['pn'][pn]: + recipe_info['description'] = event._depgraph['pn'][pn]['description'] + + if 'section' in event._depgraph['pn'][pn]: + recipe_info['section'] = event._depgraph['pn'][pn]['section'] + + if 'homepage' in event._depgraph['pn'][pn]: + recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage'] + + if 'bugtracker' in event._depgraph['pn'][pn]: + recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker'] + + recipe_info['file_path'] = file_name + recipe_info['pathflags'] = pathflags + + if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path): + recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/") + else: + raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path)) + + recipe = self.orm_wrapper.get_update_recipe_object(recipe_info) + recipe.is_image = False + if 'inherits' in event._depgraph['pn'][pn].keys(): + for cls in event._depgraph['pn'][pn]['inherits']: + if cls.endswith('/image.bbclass'): + recipe.is_image = True + recipe_info['is_image'] = True + # Save the is_image state to the relevant recipe objects + self.orm_wrapper.get_update_recipe_object(recipe_info) + break + if recipe.is_image: + for t in self.internal_state['targets']: + if pn == t.target: + t.is_image = True + t.save() + self.internal_state['recipes'][pn] = recipe + + # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED + + assume_provided = self.server.runCommand(["getVariable", "ASSUME_PROVIDED"])[0].split() + + # save recipe dependency + # buildtime + recipedeps_objects = [] + for recipe in event._depgraph['depends']: + target = self.internal_state['recipes'][recipe] + for dep in event._depgraph['depends'][recipe]: + if dep in assume_provided: + continue + via = None + if 'providermap' in event._depgraph and dep in event._depgraph['providermap']: + deprecipe = event._depgraph['providermap'][dep][0] + dependency = self.internal_state['recipes'][deprecipe] + via = Provides.objects.get_or_create(name=dep, + recipe=dependency)[0] + elif dep in self.internal_state['recipes']: + dependency = self.internal_state['recipes'][dep] + else: + errormsg += " stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep) + continue + recipe_dep = Recipe_Dependency(recipe=target, + depends_on=dependency, + via=via, + dep_type=Recipe_Dependency.TYPE_DEPENDS) + recipedeps_objects.append(recipe_dep) + + Recipe_Dependency.objects.bulk_create(recipedeps_objects) + + # save all task information + def _save_a_task(taskdesc): + spec = re.split(r'\.', taskdesc) + pn = ".".join(spec[0:-1]) + taskname = spec[-1] + e = event + e.taskname = pn + recipe = self.internal_state['recipes'][pn] + task_info = self._get_task_information(e, recipe) + task_info['task_name'] = taskname + task_obj = self.orm_wrapper.get_update_task_object(task_info) + return task_obj + + # create tasks + tasks = {} + for taskdesc in event._depgraph['tdepends']: + tasks[taskdesc] = _save_a_task(taskdesc) + + # create dependencies between tasks + taskdeps_objects = [] + for taskdesc in event._depgraph['tdepends']: + target = tasks[taskdesc] + for taskdep in event._depgraph['tdepends'][taskdesc]: + if taskdep not in tasks: + # Fetch tasks info is not collected previously + dep = _save_a_task(taskdep) + else: + dep = tasks[taskdep] + taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep )) + Task_Dependency.objects.bulk_create(taskdeps_objects) + + if len(errormsg) > 0: + logger.warn("buildinfohelper: dependency info not identify recipes: \n%s", errormsg) + + + def store_build_package_information(self, event): + package_info = BuildInfoHelper._get_data_from_event(event) + self.orm_wrapper.save_build_package_information( + self.internal_state['build'], + package_info, + self.internal_state['recipes'], + built_package=True) + + self.orm_wrapper.save_build_package_information( + self.internal_state['build'], + package_info, + self.internal_state['recipes'], + built_package=False) + + def _store_build_done(self, errorcode): + logger.info("Build exited with errorcode %d", errorcode) + br_id, be_id = self.brbe.split(":") + be = BuildEnvironment.objects.get(pk = be_id) + be.lock = BuildEnvironment.LOCK_LOCK + be.save() + br = BuildRequest.objects.get(pk = br_id) + + # if we're 'done' because we got cancelled update the build outcome + if br.state == BuildRequest.REQ_CANCELLING: + logger.info("Build cancelled") + br.build.outcome = Build.CANCELLED + br.build.save() + self.internal_state['build'] = br.build + errorcode = 0 + + if errorcode == 0: + # request archival of the project artifacts + br.state = BuildRequest.REQ_COMPLETED + else: + br.state = BuildRequest.REQ_FAILED + br.save() + + + def store_log_error(self, text): + mockevent = MockEvent() + mockevent.levelno = formatter.ERROR + mockevent.msg = text + mockevent.pathname = '-- None' + mockevent.lineno = LogMessage.ERROR + self.store_log_event(mockevent) + + def store_log_exception(self, text, backtrace = ""): + mockevent = MockEvent() + mockevent.levelno = -1 + mockevent.msg = text + mockevent.pathname = backtrace + mockevent.lineno = -1 + self.store_log_event(mockevent) + + + def store_log_event(self, event): + if event.levelno < formatter.WARNING: + return + + if 'args' in vars(event): + event.msg = event.msg % event.args + + if not 'build' in self.internal_state: + if self.brbe is None: + if not 'backlog' in self.internal_state: + self.internal_state['backlog'] = [] + self.internal_state['backlog'].append(event) + return + else: # we're under Toaster control, the build is already created + br, _ = self.brbe.split(":") + buildrequest = BuildRequest.objects.get(pk = br) + self.internal_state['build'] = buildrequest.build + + if 'build' in self.internal_state and 'backlog' in self.internal_state: + # if we have a backlog of events, do our best to save them here + if len(self.internal_state['backlog']): + tempevent = self.internal_state['backlog'].pop() + logger.debug(1, "buildinfohelper: Saving stored event %s " % tempevent) + self.store_log_event(tempevent) + else: + logger.info("buildinfohelper: All events saved") + del self.internal_state['backlog'] + + log_information = {} + log_information['build'] = self.internal_state['build'] + if event.levelno == formatter.CRITICAL: + log_information['level'] = LogMessage.CRITICAL + elif event.levelno == formatter.ERROR: + log_information['level'] = LogMessage.ERROR + elif event.levelno == formatter.WARNING: + log_information['level'] = LogMessage.WARNING + elif event.levelno == -2: # toaster self-logging + log_information['level'] = -2 + else: + log_information['level'] = LogMessage.INFO + + log_information['message'] = event.msg + log_information['pathname'] = event.pathname + log_information['lineno'] = event.lineno + logger.info("Logging error 2: %s", log_information) + + self.orm_wrapper.create_logmessage(log_information) + + def close(self, errorcode): + if self.brbe is not None: + self._store_build_done(errorcode) + + if 'backlog' in self.internal_state: + if 'build' in self.internal_state: + # we save missed events in the database for the current build + tempevent = self.internal_state['backlog'].pop() + self.store_log_event(tempevent) + else: + # we have no build, and we still have events; something amazingly wrong happend + for event in self.internal_state['backlog']: + logger.error("UNSAVED log: %s", event.msg) + + if not connection.features.autocommits_when_autocommit_is_off: + transaction.set_autocommit(True) + + # unset the brbe; this is to prevent subsequent command-line builds + # being incorrectly attached to the previous Toaster-triggered build; + # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021 + self.brbe = None diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/__init__.py new file mode 100644 index 000000000..b7cbe1a4f --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/__init__.py @@ -0,0 +1,17 @@ +# +# Gtk+ UI pieces for BitBake +# +# Copyright (C) 2006-2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/crumbsdialog.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/crumbsdialog.py new file mode 100644 index 000000000..c679f9a07 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/crumbsdialog.py @@ -0,0 +1,44 @@ +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2011-2012 Intel Corporation +# +# Authored by Joshua Lock <josh@linux.intel.com> +# Authored by Dongxiao Xu <dongxiao.xu@intel.com> +# Authored by Shane Wang <shane.wang@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import gtk + +""" +The following are convenience classes for implementing GNOME HIG compliant +BitBake GUI's +In summary: spacing = 12px, border-width = 6px +""" + +class CrumbsDialog(gtk.Dialog): + """ + A GNOME HIG compliant dialog widget. + Add buttons with gtk.Dialog.add_button or gtk.Dialog.add_buttons + """ + def __init__(self, title="", parent=None, flags=0, buttons=None): + super(CrumbsDialog, self).__init__(title, parent, flags, buttons) + + self.set_property("has-separator", False) # note: deprecated in 2.22 + + self.set_border_width(6) + self.vbox.set_property("spacing", 12) + self.action_area.set_property("spacing", 12) + self.action_area.set_property("border-width", 6) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/crumbsmessagedialog.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/crumbsmessagedialog.py new file mode 100644 index 000000000..3b998e463 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/crumbsmessagedialog.py @@ -0,0 +1,70 @@ +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2011-2012 Intel Corporation +# +# Authored by Joshua Lock <josh@linux.intel.com> +# Authored by Dongxiao Xu <dongxiao.xu@intel.com> +# Authored by Shane Wang <shane.wang@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import glib +import gtk +from bb.ui.crumbs.hobwidget import HobIconChecker +from bb.ui.crumbs.hig.crumbsdialog import CrumbsDialog + +""" +The following are convenience classes for implementing GNOME HIG compliant +BitBake GUI's +In summary: spacing = 12px, border-width = 6px +""" + +class CrumbsMessageDialog(gtk.MessageDialog): + """ + A GNOME HIG compliant dialog widget. + Add buttons with gtk.Dialog.add_button or gtk.Dialog.add_buttons + """ + def __init__(self, parent = None, label="", dialog_type = gtk.MESSAGE_QUESTION, msg=""): + super(CrumbsMessageDialog, self).__init__(None, + gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT, + dialog_type, + gtk.BUTTONS_NONE, + None) + + self.set_skip_taskbar_hint(False) + + self.set_markup(label) + + if 0 <= len(msg) < 300: + self.format_secondary_markup(msg) + else: + vbox = self.get_message_area() + vbox.set_border_width(1) + vbox.set_property("spacing", 12) + self.textWindow = gtk.ScrolledWindow() + self.textWindow.set_shadow_type(gtk.SHADOW_IN) + self.textWindow.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) + self.msgView = gtk.TextView() + self.msgView.set_editable(False) + self.msgView.set_wrap_mode(gtk.WRAP_WORD) + self.msgView.set_cursor_visible(False) + self.msgView.set_size_request(300, 300) + self.buf = gtk.TextBuffer() + self.buf.set_text(msg) + self.msgView.set_buffer(self.buf) + self.textWindow.add(self.msgView) + self.msgView.show() + vbox.add(self.textWindow) + self.textWindow.show() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/deployimagedialog.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/deployimagedialog.py new file mode 100644 index 000000000..a13fff906 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/deployimagedialog.py @@ -0,0 +1,219 @@ +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2011-2012 Intel Corporation +# +# Authored by Joshua Lock <josh@linux.intel.com> +# Authored by Dongxiao Xu <dongxiao.xu@intel.com> +# Authored by Shane Wang <shane.wang@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import glob +import gtk +import gobject +import os +import re +import shlex +import subprocess +import tempfile +from bb.ui.crumbs.hobwidget import hic, HobButton +from bb.ui.crumbs.progressbar import HobProgressBar +import bb.ui.crumbs.utils +import bb.process +from bb.ui.crumbs.hig.crumbsdialog import CrumbsDialog +from bb.ui.crumbs.hig.crumbsmessagedialog import CrumbsMessageDialog + +""" +The following are convenience classes for implementing GNOME HIG compliant +BitBake GUI's +In summary: spacing = 12px, border-width = 6px +""" + +class DeployImageDialog (CrumbsDialog): + + __dummy_usb__ = "--select a usb drive--" + + def __init__(self, title, image_path, parent, flags, buttons=None, standalone=False): + super(DeployImageDialog, self).__init__(title, parent, flags, buttons) + + self.image_path = image_path + self.standalone = standalone + + self.create_visual_elements() + self.connect("response", self.response_cb) + + def create_visual_elements(self): + self.set_size_request(600, 400) + label = gtk.Label() + label.set_alignment(0.0, 0.5) + markup = "<span font_desc='12'>The image to be written into usb drive:</span>" + label.set_markup(markup) + self.vbox.pack_start(label, expand=False, fill=False, padding=2) + + table = gtk.Table(2, 10, False) + table.set_col_spacings(5) + table.set_row_spacings(5) + self.vbox.pack_start(table, expand=True, fill=True) + + scroll = gtk.ScrolledWindow() + scroll.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC) + scroll.set_shadow_type(gtk.SHADOW_IN) + tv = gtk.TextView() + tv.set_editable(False) + tv.set_wrap_mode(gtk.WRAP_WORD) + tv.set_cursor_visible(False) + self.buf = gtk.TextBuffer() + self.buf.set_text(self.image_path) + tv.set_buffer(self.buf) + scroll.add(tv) + table.attach(scroll, 0, 10, 0, 1) + + # There are 2 ways to use DeployImageDialog + # One way is that called by HOB when the 'Deploy Image' button is clicked + # The other way is that called by a standalone script. + # Following block of codes handles the latter way. It adds a 'Select Image' button and + # emit a signal when the button is clicked. + if self.standalone: + gobject.signal_new("select_image_clicked", self, gobject.SIGNAL_RUN_FIRST, + gobject.TYPE_NONE, ()) + icon = gtk.Image() + pix_buffer = gtk.gdk.pixbuf_new_from_file(hic.ICON_IMAGES_DISPLAY_FILE) + icon.set_from_pixbuf(pix_buffer) + button = gtk.Button("Select Image") + button.set_image(icon) + #button.set_size_request(140, 50) + table.attach(button, 9, 10, 1, 2, gtk.FILL, 0, 0, 0) + button.connect("clicked", self.select_image_button_clicked_cb) + + separator = gtk.HSeparator() + self.vbox.pack_start(separator, expand=False, fill=False, padding=10) + + self.usb_desc = gtk.Label() + self.usb_desc.set_alignment(0.0, 0.5) + markup = "<span font_desc='12'>You haven't chosen any USB drive.</span>" + self.usb_desc.set_markup(markup) + + self.usb_combo = gtk.combo_box_new_text() + self.usb_combo.connect("changed", self.usb_combo_changed_cb) + model = self.usb_combo.get_model() + model.clear() + self.usb_combo.append_text(self.__dummy_usb__) + for usb in self.find_all_usb_devices(): + self.usb_combo.append_text("/dev/" + usb) + self.usb_combo.set_active(0) + self.vbox.pack_start(self.usb_combo, expand=False, fill=False) + self.vbox.pack_start(self.usb_desc, expand=False, fill=False, padding=2) + + self.progress_bar = HobProgressBar() + self.vbox.pack_start(self.progress_bar, expand=False, fill=False) + separator = gtk.HSeparator() + self.vbox.pack_start(separator, expand=False, fill=True, padding=10) + + self.vbox.show_all() + self.progress_bar.hide() + + def set_image_text_buffer(self, image_path): + self.buf.set_text(image_path) + + def set_image_path(self, image_path): + self.image_path = image_path + + def popen_read(self, cmd): + tmpout, errors = bb.process.run("%s" % cmd) + return tmpout.strip() + + def find_all_usb_devices(self): + usb_devs = [ os.readlink(u) + for u in glob.glob('/dev/disk/by-id/usb*') + if not re.search(r'part\d+', u) ] + return [ '%s' % u[u.rfind('/')+1:] for u in usb_devs ] + + def get_usb_info(self, dev): + return "%s %s" % \ + (self.popen_read('cat /sys/class/block/%s/device/vendor' % dev), + self.popen_read('cat /sys/class/block/%s/device/model' % dev)) + + def select_image_button_clicked_cb(self, button): + self.emit('select_image_clicked') + + def usb_combo_changed_cb(self, usb_combo): + combo_item = self.usb_combo.get_active_text() + if not combo_item or combo_item == self.__dummy_usb__: + markup = "<span font_desc='12'>You haven't chosen any USB drive.</span>" + self.usb_desc.set_markup(markup) + else: + markup = "<span font_desc='12'>" + self.get_usb_info(combo_item.lstrip("/dev/")) + "</span>" + self.usb_desc.set_markup(markup) + + def response_cb(self, dialog, response_id): + if response_id == gtk.RESPONSE_YES: + lbl = '' + msg = '' + combo_item = self.usb_combo.get_active_text() + if combo_item and combo_item != self.__dummy_usb__ and self.image_path: + cmdline = bb.ui.crumbs.utils.which_terminal() + if cmdline: + tmpfile = tempfile.NamedTemporaryFile() + cmdline += "\"sudo dd if=" + self.image_path + \ + " of=" + combo_item + " && sync; echo $? > " + tmpfile.name + "\"" + subprocess.call(shlex.split(cmdline)) + + if int(tmpfile.readline().strip()) == 0: + lbl = "<b>Deploy image successfully.</b>" + else: + lbl = "<b>Failed to deploy image.</b>" + msg = "Please check image <b>%s</b> exists and USB device <b>%s</b> is writable." % (self.image_path, combo_item) + tmpfile.close() + else: + if not self.image_path: + lbl = "<b>No selection made.</b>" + msg = "You have not selected an image to deploy." + else: + lbl = "<b>No selection made.</b>" + msg = "You have not selected a USB device." + if len(lbl): + crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_INFO, msg) + button = crumbs_dialog.add_button("Close", gtk.RESPONSE_OK) + HobButton.style_button(button) + crumbs_dialog.run() + crumbs_dialog.destroy() + + def update_progress_bar(self, title, fraction, status=None): + self.progress_bar.update(fraction) + self.progress_bar.set_title(title) + self.progress_bar.set_rcstyle(status) + + def write_file(self, ifile, ofile): + self.progress_bar.reset() + self.progress_bar.show() + + f_from = os.open(ifile, os.O_RDONLY) + f_to = os.open(ofile, os.O_WRONLY) + + total_size = os.stat(ifile).st_size + written_size = 0 + + while True: + buf = os.read(f_from, 1024*1024) + if not buf: + break + os.write(f_to, buf) + written_size += 1024*1024 + self.update_progress_bar("Writing to usb:", written_size * 1.0/total_size) + + self.update_progress_bar("Writing completed:", 1.0) + os.close(f_from) + os.close(f_to) + self.progress_bar.hide() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/imageselectiondialog.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/imageselectiondialog.py new file mode 100644 index 000000000..21216adc9 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/imageselectiondialog.py @@ -0,0 +1,172 @@ +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2011-2012 Intel Corporation +# +# Authored by Joshua Lock <josh@linux.intel.com> +# Authored by Dongxiao Xu <dongxiao.xu@intel.com> +# Authored by Shane Wang <shane.wang@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import gtk +import gobject +import os +from bb.ui.crumbs.hobwidget import HobViewTable, HobInfoButton, HobButton, HobAltButton +from bb.ui.crumbs.hig.crumbsdialog import CrumbsDialog +from bb.ui.crumbs.hig.layerselectiondialog import LayerSelectionDialog + +""" +The following are convenience classes for implementing GNOME HIG compliant +BitBake GUI's +In summary: spacing = 12px, border-width = 6px +""" + +class ImageSelectionDialog (CrumbsDialog): + + __columns__ = [{ + 'col_name' : 'Image name', + 'col_id' : 0, + 'col_style': 'text', + 'col_min' : 400, + 'col_max' : 400 + }, { + 'col_name' : 'Select', + 'col_id' : 1, + 'col_style': 'radio toggle', + 'col_min' : 160, + 'col_max' : 160 + }] + + + def __init__(self, image_folder, image_types, title, parent, flags, buttons=None, image_extension = {}): + super(ImageSelectionDialog, self).__init__(title, parent, flags, buttons) + self.connect("response", self.response_cb) + + self.image_folder = image_folder + self.image_types = image_types + self.image_list = [] + self.image_names = [] + self.image_extension = image_extension + + # create visual elements on the dialog + self.create_visual_elements() + + self.image_store = gtk.ListStore(gobject.TYPE_STRING, gobject.TYPE_BOOLEAN) + self.fill_image_store() + + def create_visual_elements(self): + hbox = gtk.HBox(False, 6) + + self.vbox.pack_start(hbox, expand=False, fill=False) + + entry = gtk.Entry() + entry.set_text(self.image_folder) + table = gtk.Table(1, 10, True) + table.set_size_request(560, -1) + hbox.pack_start(table, expand=False, fill=False) + table.attach(entry, 0, 9, 0, 1) + image = gtk.Image() + image.set_from_stock(gtk.STOCK_OPEN, gtk.ICON_SIZE_BUTTON) + open_button = gtk.Button() + open_button.set_image(image) + open_button.connect("clicked", self.select_path_cb, self, entry) + table.attach(open_button, 9, 10, 0, 1) + + self.image_table = HobViewTable(self.__columns__, "Images") + self.image_table.set_size_request(-1, 300) + self.image_table.connect("toggled", self.toggled_cb) + self.image_table.connect_group_selection(self.table_selected_cb) + self.image_table.connect("row-activated", self.row_actived_cb) + self.vbox.pack_start(self.image_table, expand=True, fill=True) + + self.show_all() + + def change_image_cb(self, model, path, columnid): + if not model: + return + iter = model.get_iter_first() + while iter: + rowpath = model.get_path(iter) + model[rowpath][columnid] = False + iter = model.iter_next(iter) + + model[path][columnid] = True + + def toggled_cb(self, table, cell, path, columnid, tree): + model = tree.get_model() + self.change_image_cb(model, path, columnid) + + def table_selected_cb(self, selection): + model, paths = selection.get_selected_rows() + if paths: + self.change_image_cb(model, paths[0], 1) + + def row_actived_cb(self, tab, model, path): + self.change_image_cb(model, path, 1) + self.emit('response', gtk.RESPONSE_YES) + + def select_path_cb(self, action, parent, entry): + dialog = gtk.FileChooserDialog("", parent, + gtk.FILE_CHOOSER_ACTION_SELECT_FOLDER) + text = entry.get_text() + dialog.set_current_folder(text if len(text) > 0 else os.getcwd()) + button = dialog.add_button("Cancel", gtk.RESPONSE_NO) + HobAltButton.style_button(button) + button = dialog.add_button("Open", gtk.RESPONSE_YES) + HobButton.style_button(button) + response = dialog.run() + if response == gtk.RESPONSE_YES: + path = dialog.get_filename() + entry.set_text(path) + self.image_folder = path + self.fill_image_store() + + dialog.destroy() + + def fill_image_store(self): + self.image_list = [] + self.image_store.clear() + imageset = set() + for root, dirs, files in os.walk(self.image_folder): + # ignore the sub directories + dirs[:] = [] + for f in files: + for image_type in self.image_types: + if image_type in self.image_extension: + real_types = self.image_extension[image_type] + else: + real_types = [image_type] + for real_image_type in real_types: + if f.endswith('.' + real_image_type): + imageset.add(f.rsplit('.' + real_image_type)[0].rsplit('.rootfs')[0]) + self.image_list.append(f) + + for image in imageset: + self.image_store.set(self.image_store.append(), 0, image, 1, False) + + self.image_table.set_model(self.image_store) + + def response_cb(self, dialog, response_id): + self.image_names = [] + if response_id == gtk.RESPONSE_YES: + iter = self.image_store.get_iter_first() + while iter: + path = self.image_store.get_path(iter) + if self.image_store[path][1]: + for f in self.image_list: + if f.startswith(self.image_store[path][0] + '.'): + self.image_names.append(f) + break + iter = self.image_store.iter_next(iter) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/layerselectiondialog.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/layerselectiondialog.py new file mode 100644 index 000000000..52d57b673 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/layerselectiondialog.py @@ -0,0 +1,298 @@ +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2011-2012 Intel Corporation +# +# Authored by Joshua Lock <josh@linux.intel.com> +# Authored by Dongxiao Xu <dongxiao.xu@intel.com> +# Authored by Shane Wang <shane.wang@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import gtk +import gobject +import os +import tempfile +from bb.ui.crumbs.hobwidget import hic, HobButton, HobAltButton +from bb.ui.crumbs.hig.crumbsdialog import CrumbsDialog +from bb.ui.crumbs.hig.crumbsmessagedialog import CrumbsMessageDialog + +""" +The following are convenience classes for implementing GNOME HIG compliant +BitBake GUI's +In summary: spacing = 12px, border-width = 6px +""" + +class CellRendererPixbufActivatable(gtk.CellRendererPixbuf): + """ + A custom CellRenderer implementation which is activatable + so that we can handle user clicks + """ + __gsignals__ = { 'clicked' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_STRING,)), } + + def __init__(self): + gtk.CellRendererPixbuf.__init__(self) + self.set_property('mode', gtk.CELL_RENDERER_MODE_ACTIVATABLE) + self.set_property('follow-state', True) + + """ + Respond to a user click on a cell + """ + def do_activate(self, even, widget, path, background_area, cell_area, flags): + self.emit('clicked', path) + +# +# LayerSelectionDialog +# +class LayerSelectionDialog (CrumbsDialog): + + TARGETS = [ + ("MY_TREE_MODEL_ROW", gtk.TARGET_SAME_WIDGET, 0), + ("text/plain", 0, 1), + ("TEXT", 0, 2), + ("STRING", 0, 3), + ] + + def gen_label_widget(self, content): + label = gtk.Label() + label.set_alignment(0, 0) + label.set_markup(content) + label.show() + return label + + def layer_widget_toggled_cb(self, cell, path, layer_store): + name = layer_store[path][0] + toggle = not layer_store[path][1] + layer_store[path][1] = toggle + + def layer_widget_add_clicked_cb(self, action, layer_store, parent): + dialog = gtk.FileChooserDialog("Add new layer", parent, + gtk.FILE_CHOOSER_ACTION_SELECT_FOLDER) + button = dialog.add_button("Cancel", gtk.RESPONSE_NO) + HobAltButton.style_button(button) + button = dialog.add_button("Open", gtk.RESPONSE_YES) + HobButton.style_button(button) + label = gtk.Label("Select the layer you wish to add") + label.show() + dialog.set_extra_widget(label) + response = dialog.run() + path = dialog.get_filename() + dialog.destroy() + + lbl = "<b>Error</b>" + msg = "Unable to load layer <i>%s</i> because " % path + if response == gtk.RESPONSE_YES: + import os + import os.path + layers = [] + it = layer_store.get_iter_first() + while it: + layers.append(layer_store.get_value(it, 0)) + it = layer_store.iter_next(it) + + if not path: + msg += "it is an invalid path." + elif not os.path.exists(path+"/conf/layer.conf"): + msg += "there is no layer.conf inside the directory." + elif path in layers: + msg += "it is already in loaded layers." + else: + layer_store.append([path]) + return + dialog = CrumbsMessageDialog(parent, lbl, gtk.MESSAGE_ERROR, msg) + dialog.add_button(gtk.STOCK_CLOSE, gtk.RESPONSE_OK) + response = dialog.run() + dialog.destroy() + + def layer_widget_del_clicked_cb(self, action, tree_selection, layer_store): + model, iter = tree_selection.get_selected() + if iter: + layer_store.remove(iter) + + + def gen_layer_widget(self, layers, layers_avail, window, tooltip=""): + hbox = gtk.HBox(False, 6) + + layer_tv = gtk.TreeView() + layer_tv.set_rules_hint(True) + layer_tv.set_headers_visible(False) + tree_selection = layer_tv.get_selection() + tree_selection.set_mode(gtk.SELECTION_SINGLE) + + # Allow enable drag and drop of rows including row move + dnd_internal_target = '' + dnd_targets = [(dnd_internal_target, gtk.TARGET_SAME_WIDGET, 0)] + layer_tv.enable_model_drag_source( gtk.gdk.BUTTON1_MASK, + dnd_targets, + gtk.gdk.ACTION_MOVE) + layer_tv.enable_model_drag_dest(dnd_targets, + gtk.gdk.ACTION_MOVE) + layer_tv.connect("drag_data_get", self.drag_data_get_cb) + layer_tv.connect("drag_data_received", self.drag_data_received_cb) + + col0= gtk.TreeViewColumn('Path') + cell0 = gtk.CellRendererText() + cell0.set_padding(5,2) + col0.pack_start(cell0, True) + col0.set_cell_data_func(cell0, self.draw_layer_path_cb) + layer_tv.append_column(col0) + + scroll = gtk.ScrolledWindow() + scroll.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC) + scroll.set_shadow_type(gtk.SHADOW_IN) + scroll.add(layer_tv) + + table_layer = gtk.Table(2, 10, False) + hbox.pack_start(table_layer, expand=True, fill=True) + + table_layer.attach(scroll, 0, 10, 0, 1) + + layer_store = gtk.ListStore(gobject.TYPE_STRING) + for layer in layers: + layer_store.append([layer]) + + col1 = gtk.TreeViewColumn('Enabled') + layer_tv.append_column(col1) + + cell1 = CellRendererPixbufActivatable() + cell1.set_fixed_size(-1,35) + cell1.connect("clicked", self.del_cell_clicked_cb, layer_store) + col1.pack_start(cell1, True) + col1.set_cell_data_func(cell1, self.draw_delete_button_cb, layer_tv) + + add_button = gtk.Button() + add_button.set_relief(gtk.RELIEF_NONE) + box = gtk.HBox(False, 6) + box.show() + add_button.add(box) + add_button.connect("enter-notify-event", self.add_hover_cb) + add_button.connect("leave-notify-event", self.add_leave_cb) + self.im = gtk.Image() + self.im.set_from_file(hic.ICON_INDI_ADD_FILE) + self.im.show() + box.pack_start(self.im, expand=False, fill=False, padding=6) + lbl = gtk.Label("Add layer") + lbl.set_alignment(0.0, 0.5) + lbl.show() + box.pack_start(lbl, expand=True, fill=True, padding=6) + add_button.connect("clicked", self.layer_widget_add_clicked_cb, layer_store, window) + table_layer.attach(add_button, 0, 10, 1, 2, gtk.EXPAND | gtk.FILL, 0, 0, 6) + layer_tv.set_model(layer_store) + + hbox.show_all() + + return hbox, layer_store + + def drag_data_get_cb(self, treeview, context, selection, target_id, etime): + treeselection = treeview.get_selection() + model, iter = treeselection.get_selected() + data = model.get_value(iter, 0) + selection.set(selection.target, 8, data) + + def drag_data_received_cb(self, treeview, context, x, y, selection, info, etime): + model = treeview.get_model() + data = selection.data + drop_info = treeview.get_dest_row_at_pos(x, y) + if drop_info: + path, position = drop_info + iter = model.get_iter(path) + if (position == gtk.TREE_VIEW_DROP_BEFORE or position == gtk.TREE_VIEW_DROP_INTO_OR_BEFORE): + model.insert_before(iter, [data]) + else: + model.insert_after(iter, [data]) + else: + model.append([data]) + if context.action == gtk.gdk.ACTION_MOVE: + context.finish(True, True, etime) + return + + def add_hover_cb(self, button, event): + self.im.set_from_file(hic.ICON_INDI_ADD_HOVER_FILE) + + def add_leave_cb(self, button, event): + self.im.set_from_file(hic.ICON_INDI_ADD_FILE) + + def __init__(self, title, layers, layers_non_removable, all_layers, parent, flags, buttons=None): + super(LayerSelectionDialog, self).__init__(title, parent, flags, buttons) + + # class members from other objects + self.layers = layers + self.layers_non_removable = layers_non_removable + self.all_layers = all_layers + self.layers_changed = False + + # icon for remove button in TreeView + im = gtk.Image() + im.set_from_file(hic.ICON_INDI_REMOVE_FILE) + self.rem_icon = im.get_pixbuf() + + # class members for internal use + self.layer_store = None + + # create visual elements on the dialog + self.create_visual_elements() + self.connect("response", self.response_cb) + + def create_visual_elements(self): + layer_widget, self.layer_store = self.gen_layer_widget(self.layers, self.all_layers, self, None) + layer_widget.set_size_request(450, 250) + self.vbox.pack_start(layer_widget, expand=True, fill=True) + self.show_all() + + def response_cb(self, dialog, response_id): + model = self.layer_store + it = model.get_iter_first() + layers = [] + while it: + layers.append(model.get_value(it, 0)) + it = model.iter_next(it) + + self.layers_changed = (self.layers != layers) + self.layers = layers + + """ + A custom cell_data_func to draw a delete 'button' in the TreeView for layers + other than the meta layer. The deletion of which is prevented so that the + user can't shoot themselves in the foot too badly. + """ + def draw_delete_button_cb(self, col, cell, model, it, tv): + path = model.get_value(it, 0) + if path in self.layers_non_removable: + cell.set_sensitive(False) + cell.set_property('pixbuf', None) + cell.set_property('mode', gtk.CELL_RENDERER_MODE_INERT) + else: + cell.set_property('pixbuf', self.rem_icon) + cell.set_sensitive(True) + cell.set_property('mode', gtk.CELL_RENDERER_MODE_ACTIVATABLE) + + return True + + """ + A custom cell_data_func to write an extra message into the layer path cell + for the meta layer. We should inform the user that they can't remove it for + their own safety. + """ + def draw_layer_path_cb(self, col, cell, model, it): + path = model.get_value(it, 0) + if path in self.layers_non_removable: + cell.set_property('markup', "<b>It cannot be removed</b>\n%s" % path) + else: + cell.set_property('text', path) + + def del_cell_clicked_cb(self, cell, path, model): + it = model.get_iter_from_string(path) + model.remove(it) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/propertydialog.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/propertydialog.py new file mode 100644 index 000000000..09b9ce6de --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/propertydialog.py @@ -0,0 +1,437 @@ +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2011-2013 Intel Corporation +# +# Authored by Andrei Dinu <andrei.adrianx.dinu@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import string +import gtk +import gobject +import os +import tempfile +import glib +from bb.ui.crumbs.hig.crumbsdialog import CrumbsDialog +from bb.ui.crumbs.hig.settingsuihelper import SettingsUIHelper +from bb.ui.crumbs.hig.crumbsmessagedialog import CrumbsMessageDialog +from bb.ui.crumbs.hig.layerselectiondialog import LayerSelectionDialog + +""" +The following are convenience classes for implementing GNOME HIG compliant +BitBake GUI's +In summary: spacing = 12px, border-width = 6px +""" + +class PropertyDialog(CrumbsDialog): + + def __init__(self, title, parent, information, flags, buttons=None): + + super(PropertyDialog, self).__init__(title, parent, flags, buttons) + + self.properties = information + + if len(self.properties) == 10: + self.create_recipe_visual_elements() + elif len(self.properties) == 5: + self.create_package_visual_elements() + else: + self.create_information_visual_elements() + + + def create_information_visual_elements(self): + + HOB_ICON_BASE_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ("icons/")) + ICON_PACKAGES_DISPLAY_FILE = os.path.join(HOB_ICON_BASE_DIR, ('info/info_display.png')) + + self.set_resizable(False) + + self.table = gtk.Table(1,1,False) + self.table.set_row_spacings(0) + self.table.set_col_spacings(0) + + self.image = gtk.Image() + self.image.set_from_file(ICON_PACKAGES_DISPLAY_FILE) + self.image.set_property("xalign",0) + #self.vbox.add(self.image) + + image_info = self.properties.split("*")[0] + info = self.properties.split("*")[1] + + vbox = gtk.VBox(True, spacing=30) + + self.label_short = gtk.Label() + self.label_short.set_line_wrap(False) + self.label_short.set_markup(image_info) + self.label_short.set_property("xalign", 0) + + self.info_label = gtk.Label() + self.info_label.set_line_wrap(True) + self.info_label.set_markup(info) + self.info_label.set_property("yalign", 0.5) + + self.table.attach(self.image, 0,1,0,1, xoptions=gtk.FILL|gtk.EXPAND, yoptions=gtk.FILL,xpadding=5,ypadding=5) + self.table.attach(self.label_short, 0,1,0,1, xoptions=gtk.FILL|gtk.EXPAND, yoptions=gtk.FILL,xpadding=40,ypadding=5) + self.table.attach(self.info_label, 0,1,1,2, xoptions=gtk.FILL|gtk.EXPAND, yoptions=gtk.FILL,xpadding=40,ypadding=10) + + self.vbox.add(self.table) + self.connect('delete-event', lambda w, e: self.destroy() or True) + + def treeViewTooltip( self, widget, e, tooltips, cell, emptyText="" ): + try: + (path,col,x,y) = widget.get_path_at_pos( int(e.x), int(e.y) ) + it = widget.get_model().get_iter(path) + value = widget.get_model().get_value(it,cell) + if value in self.tooltip_items: + tooltips.set_tip(widget, self.tooltip_items[value]) + tooltips.enable() + else: + tooltips.set_tip(widget, emptyText) + except: + tooltips.set_tip(widget, emptyText) + + + def create_package_visual_elements(self): + + import json + + name = self.properties['name'] + binb = self.properties['binb'] + size = self.properties['size'] + recipe = self.properties['recipe'] + file_list = json.loads(self.properties['files_list']) + + files_temp = '' + paths_temp = '' + files_binb = [] + paths_binb = [] + + self.tooltip_items = {} + + self.set_resizable(False) + + #cleaning out the recipe variable + recipe = recipe.split("+")[0] + + vbox = gtk.VBox(True,spacing = 0) + + ###################################### NAME ROW + COL ################################# + + self.label_short = gtk.Label() + self.label_short.set_size_request(300,-1) + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<span weight=\"bold\">Name: </span>" + name) + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + + ###################################### SIZE ROW + COL ###################################### + + self.label_short = gtk.Label() + self.label_short.set_size_request(300,-1) + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<span weight=\"bold\">Size: </span>" + size) + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + + ##################################### RECIPE ROW + COL ######################################### + + self.label_short = gtk.Label() + self.label_short.set_size_request(300,-1) + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<span weight=\"bold\">Recipe: </span>" + recipe) + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + + ##################################### BINB ROW + COL ####################################### + + if binb != '': + self.label_short = gtk.Label() + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<span weight=\"bold\">Brought in by: </span>") + self.label_short.set_property("xalign", 0) + + self.label_info = gtk.Label() + self.label_info.set_size_request(300,-1) + self.label_info.set_selectable(True) + self.label_info.set_line_wrap(True) + self.label_info.set_markup(binb) + self.label_info.set_property("xalign", 0) + + self.vbox.add(self.label_short) + self.vbox.add(self.label_info) + + #################################### FILES BROUGHT BY PACKAGES ################################### + + if file_list: + + self.textWindow = gtk.ScrolledWindow() + self.textWindow.set_shadow_type(gtk.SHADOW_IN) + self.textWindow.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) + self.textWindow.set_size_request(100, 170) + + packagefiles_store = gtk.ListStore(str) + + self.packagefiles_tv = gtk.TreeView() + self.packagefiles_tv.set_rules_hint(True) + self.packagefiles_tv.set_headers_visible(True) + self.textWindow.add(self.packagefiles_tv) + + self.cell1 = gtk.CellRendererText() + col1 = gtk.TreeViewColumn('Package files', self.cell1) + col1.set_cell_data_func(self.cell1, self.regex_field) + self.packagefiles_tv.append_column(col1) + + items = file_list.keys() + items.sort() + for item in items: + fullpath = item + while len(item) > 35: + item = item[:len(item)/2] + "" + item[len(item)/2+1:] + if len(item) == 35: + item = item[:len(item)/2] + "..." + item[len(item)/2+3:] + self.tooltip_items[item] = fullpath + + packagefiles_store.append([str(item)]) + + self.packagefiles_tv.set_model(packagefiles_store) + + tips = gtk.Tooltips() + tips.set_tip(self.packagefiles_tv, "") + self.packagefiles_tv.connect("motion-notify-event", self.treeViewTooltip, tips, 0) + self.packagefiles_tv.set_events(gtk.gdk.POINTER_MOTION_MASK) + + self.vbox.add(self.textWindow) + + self.vbox.show_all() + + + def regex_field(self, column, cell, model, iter): + cell.set_property('text', model.get_value(iter, 0)) + return + + + def create_recipe_visual_elements(self): + + summary = self.properties['summary'] + name = self.properties['name'] + version = self.properties['version'] + revision = self.properties['revision'] + binb = self.properties['binb'] + group = self.properties['group'] + license = self.properties['license'] + homepage = self.properties['homepage'] + bugtracker = self.properties['bugtracker'] + description = self.properties['description'] + + self.set_resizable(False) + + #cleaning out the version variable and also the summary + version = version.split(":")[1] + if len(version) > 30: + version = version.split("+")[0] + else: + version = version.split("-")[0] + license = license.replace("&" , "and") + if (homepage == ''): + homepage = 'unknown' + if (bugtracker == ''): + bugtracker = 'unknown' + summary = summary.split("+")[0] + + #calculating the rows needed for the table + binb_items_count = len(binb.split(',')) + binb_items = binb.split(',') + + vbox = gtk.VBox(False,spacing = 0) + + ######################################## SUMMARY LABEL ######################################### + + if summary != '': + self.label_short = gtk.Label() + self.label_short.set_width_chars(37) + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<b>" + summary + "</b>") + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + + ########################################## NAME ROW + COL ####################################### + + self.label_short = gtk.Label() + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<span weight=\"bold\">Name: </span>" + name) + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + + ####################################### VERSION ROW + COL #################################### + + self.label_short = gtk.Label() + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<span weight=\"bold\">Version: </span>" + version) + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + + ##################################### REVISION ROW + COL ##################################### + + self.label_short = gtk.Label() + self.label_short.set_line_wrap(True) + self.label_short.set_selectable(True) + self.label_short.set_markup("<span weight=\"bold\">Revision: </span>" + revision) + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + + ################################## GROUP ROW + COL ############################################ + + self.label_short = gtk.Label() + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<span weight=\"bold\">Group: </span>" + group) + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + + ################################# HOMEPAGE ROW + COL ############################################ + + if homepage != 'unknown': + self.label_info = gtk.Label() + self.label_info.set_selectable(True) + self.label_info.set_line_wrap(True) + if len(homepage) > 35: + self.label_info.set_markup("<a href=\"" + homepage + "\">" + homepage[0:35] + "..." + "</a>") + else: + self.label_info.set_markup("<a href=\"" + homepage + "\">" + homepage[0:60] + "</a>") + + self.label_info.set_property("xalign", 0) + + self.label_short = gtk.Label() + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<b>Homepage: </b>") + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + self.vbox.add(self.label_info) + + ################################# BUGTRACKER ROW + COL ########################################### + + if bugtracker != 'unknown': + self.label_info = gtk.Label() + self.label_info.set_selectable(True) + self.label_info.set_line_wrap(True) + if len(bugtracker) > 35: + self.label_info.set_markup("<a href=\"" + bugtracker + "\">" + bugtracker[0:35] + "..." + "</a>") + else: + self.label_info.set_markup("<a href=\"" + bugtracker + "\">" + bugtracker[0:60] + "</a>") + self.label_info.set_property("xalign", 0) + + self.label_short = gtk.Label() + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<b>Bugtracker: </b>") + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + self.vbox.add(self.label_info) + + ################################# LICENSE ROW + COL ############################################ + + self.label_info = gtk.Label() + self.label_info.set_selectable(True) + self.label_info.set_line_wrap(True) + self.label_info.set_markup(license) + self.label_info.set_property("xalign", 0) + + self.label_short = gtk.Label() + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<span weight=\"bold\">License: </span>") + self.label_short.set_property("xalign", 0) + + self.vbox.add(self.label_short) + self.vbox.add(self.label_info) + + ################################### BINB ROW+COL ############################################# + + if binb != '': + self.label_short = gtk.Label() + self.label_short.set_selectable(True) + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<span weight=\"bold\">Brought in by: </span>") + self.label_short.set_property("xalign", 0) + self.vbox.add(self.label_short) + self.label_info = gtk.Label() + self.label_info.set_selectable(True) + self.label_info.set_width_chars(36) + if len(binb) > 200: + scrolled_window = gtk.ScrolledWindow() + scrolled_window.set_policy(gtk.POLICY_NEVER,gtk.POLICY_ALWAYS) + scrolled_window.set_size_request(100,100) + self.label_info.set_markup(binb) + self.label_info.set_padding(6,6) + self.label_info.set_alignment(0,0) + self.label_info.set_line_wrap(True) + scrolled_window.add_with_viewport(self.label_info) + self.vbox.add(scrolled_window) + else: + self.label_info.set_markup(binb) + self.label_info.set_property("xalign", 0) + self.label_info.set_line_wrap(True) + self.vbox.add(self.label_info) + + ################################ DESCRIPTION TAG ROW ################################################# + + self.label_short = gtk.Label() + self.label_short.set_line_wrap(True) + self.label_short.set_markup("<span weight=\"bold\">Description </span>") + self.label_short.set_property("xalign", 0) + self.vbox.add(self.label_short) + + ################################ DESCRIPTION INFORMATION ROW ########################################## + + hbox = gtk.HBox(True,spacing = 0) + + self.label_short = gtk.Label() + self.label_short.set_selectable(True) + self.label_short.set_width_chars(36) + if len(description) > 200: + scrolled_window = gtk.ScrolledWindow() + scrolled_window.set_policy(gtk.POLICY_NEVER,gtk.POLICY_ALWAYS) + scrolled_window.set_size_request(100,100) + self.label_short.set_markup(description) + self.label_short.set_padding(6,6) + self.label_short.set_alignment(0,0) + self.label_short.set_line_wrap(True) + scrolled_window.add_with_viewport(self.label_short) + self.vbox.add(scrolled_window) + else: + self.label_short.set_markup(description) + self.label_short.set_property("xalign", 0) + self.label_short.set_line_wrap(True) + self.vbox.add(self.label_short) + + self.vbox.show_all() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/settingsuihelper.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/settingsuihelper.py new file mode 100644 index 000000000..e0285c93c --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hig/settingsuihelper.py @@ -0,0 +1,122 @@ +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2011-2012 Intel Corporation +# +# Authored by Joshua Lock <josh@linux.intel.com> +# Authored by Dongxiao Xu <dongxiao.xu@intel.com> +# Authored by Shane Wang <shane.wang@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import gtk +import os +from bb.ui.crumbs.hobwidget import HobInfoButton, HobButton, HobAltButton + +""" +The following are convenience classes for implementing GNOME HIG compliant +BitBake GUI's +In summary: spacing = 12px, border-width = 6px +""" + +class SettingsUIHelper(): + + def gen_label_widget(self, content): + label = gtk.Label() + label.set_alignment(0, 0) + label.set_markup(content) + label.show() + return label + + def gen_label_info_widget(self, content, tooltip): + table = gtk.Table(1, 10, False) + label = self.gen_label_widget(content) + info = HobInfoButton(tooltip, self) + table.attach(label, 0, 1, 0, 1, xoptions=gtk.FILL) + table.attach(info, 1, 2, 0, 1, xoptions=gtk.FILL, xpadding=10) + return table + + def gen_spinner_widget(self, content, lower, upper, tooltip=""): + hbox = gtk.HBox(False, 12) + adjust = gtk.Adjustment(value=content, lower=lower, upper=upper, step_incr=1) + spinner = gtk.SpinButton(adjustment=adjust, climb_rate=1, digits=0) + + spinner.set_value(content) + hbox.pack_start(spinner, expand=False, fill=False) + + info = HobInfoButton(tooltip, self) + hbox.pack_start(info, expand=False, fill=False) + + hbox.show_all() + return hbox, spinner + + def gen_combo_widget(self, curr_item, all_item, tooltip=""): + hbox = gtk.HBox(False, 12) + combo = gtk.combo_box_new_text() + hbox.pack_start(combo, expand=False, fill=False) + + index = 0 + for item in all_item or []: + combo.append_text(item) + if item == curr_item: + combo.set_active(index) + index += 1 + + info = HobInfoButton(tooltip, self) + hbox.pack_start(info, expand=False, fill=False) + + hbox.show_all() + return hbox, combo + + def entry_widget_select_path_cb(self, action, parent, entry): + dialog = gtk.FileChooserDialog("", parent, + gtk.FILE_CHOOSER_ACTION_SELECT_FOLDER) + text = entry.get_text() + dialog.set_current_folder(text if len(text) > 0 else os.getcwd()) + button = dialog.add_button("Cancel", gtk.RESPONSE_NO) + HobAltButton.style_button(button) + button = dialog.add_button("Open", gtk.RESPONSE_YES) + HobButton.style_button(button) + response = dialog.run() + if response == gtk.RESPONSE_YES: + path = dialog.get_filename() + entry.set_text(path) + + dialog.destroy() + + def gen_entry_widget(self, content, parent, tooltip="", need_button=True): + hbox = gtk.HBox(False, 12) + entry = gtk.Entry() + entry.set_text(content) + entry.set_size_request(350,30) + + if need_button: + table = gtk.Table(1, 10, False) + hbox.pack_start(table, expand=True, fill=True) + table.attach(entry, 0, 9, 0, 1, xoptions=gtk.SHRINK) + image = gtk.Image() + image.set_from_stock(gtk.STOCK_OPEN,gtk.ICON_SIZE_BUTTON) + open_button = gtk.Button() + open_button.set_image(image) + open_button.connect("clicked", self.entry_widget_select_path_cb, parent, entry) + table.attach(open_button, 9, 10, 0, 1, xoptions=gtk.SHRINK) + else: + hbox.pack_start(entry, expand=True, fill=True) + + if tooltip != "": + info = HobInfoButton(tooltip, self) + hbox.pack_start(info, expand=False, fill=False) + + hbox.show_all() + return hbox, entry diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hobcolor.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hobcolor.py new file mode 100644 index 000000000..3316542a2 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hobcolor.py @@ -0,0 +1,38 @@ +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2012 Intel Corporation +# +# Authored by Shane Wang <shane.wang@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +class HobColors: + WHITE = "#ffffff" + PALE_GREEN = "#aaffaa" + ORANGE = "#eb8e68" + PALE_RED = "#ffaaaa" + GRAY = "#aaaaaa" + LIGHT_GRAY = "#dddddd" + SLIGHT_DARK = "#5f5f5f" + DARK = "#3c3b37" + BLACK = "#000000" + PALE_BLUE = "#53b8ff" + DEEP_RED = "#aa3e3e" + KHAKI = "#fff68f" + + OK = WHITE + RUNNING = PALE_GREEN + WARNING = ORANGE + ERROR = PALE_RED diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hobwidget.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hobwidget.py new file mode 100644 index 000000000..2b969c146 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/hobwidget.py @@ -0,0 +1,904 @@ +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2011-2012 Intel Corporation +# +# Authored by Dongxiao Xu <dongxiao.xu@intel.com> +# Authored by Shane Wang <shane.wang@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import gtk +import gobject +import os +import os.path +import sys +import pango, pangocairo +import cairo +import math + +from bb.ui.crumbs.hobcolor import HobColors +from bb.ui.crumbs.persistenttooltip import PersistentTooltip + +class hwc: + + MAIN_WIN_WIDTH = 1024 + MAIN_WIN_HEIGHT = 700 + +class hic: + + HOB_ICON_BASE_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ("ui/icons/")) + + ICON_RCIPE_DISPLAY_FILE = os.path.join(HOB_ICON_BASE_DIR, ('recipe/recipe_display.png')) + ICON_RCIPE_HOVER_FILE = os.path.join(HOB_ICON_BASE_DIR, ('recipe/recipe_hover.png')) + ICON_PACKAGES_DISPLAY_FILE = os.path.join(HOB_ICON_BASE_DIR, ('packages/packages_display.png')) + ICON_PACKAGES_HOVER_FILE = os.path.join(HOB_ICON_BASE_DIR, ('packages/packages_hover.png')) + ICON_LAYERS_DISPLAY_FILE = os.path.join(HOB_ICON_BASE_DIR, ('layers/layers_display.png')) + ICON_LAYERS_HOVER_FILE = os.path.join(HOB_ICON_BASE_DIR, ('layers/layers_hover.png')) + ICON_IMAGES_DISPLAY_FILE = os.path.join(HOB_ICON_BASE_DIR, ('images/images_display.png')) + ICON_IMAGES_HOVER_FILE = os.path.join(HOB_ICON_BASE_DIR, ('images/images_hover.png')) + ICON_SETTINGS_DISPLAY_FILE = os.path.join(HOB_ICON_BASE_DIR, ('settings/settings_display.png')) + ICON_SETTINGS_HOVER_FILE = os.path.join(HOB_ICON_BASE_DIR, ('settings/settings_hover.png')) + ICON_INFO_DISPLAY_FILE = os.path.join(HOB_ICON_BASE_DIR, ('info/info_display.png')) + ICON_INFO_HOVER_FILE = os.path.join(HOB_ICON_BASE_DIR, ('info/info_hover.png')) + ICON_INDI_CONFIRM_FILE = os.path.join(HOB_ICON_BASE_DIR, ('indicators/confirmation.png')) + ICON_INDI_ERROR_FILE = os.path.join(HOB_ICON_BASE_DIR, ('indicators/denied.png')) + ICON_INDI_REMOVE_FILE = os.path.join(HOB_ICON_BASE_DIR, ('indicators/remove.png')) + ICON_INDI_REMOVE_HOVER_FILE = os.path.join(HOB_ICON_BASE_DIR, ('indicators/remove-hover.png')) + ICON_INDI_ADD_FILE = os.path.join(HOB_ICON_BASE_DIR, ('indicators/add.png')) + ICON_INDI_ADD_HOVER_FILE = os.path.join(HOB_ICON_BASE_DIR, ('indicators/add-hover.png')) + ICON_INDI_REFRESH_FILE = os.path.join(HOB_ICON_BASE_DIR, ('indicators/refresh.png')) + ICON_INDI_ALERT_FILE = os.path.join(HOB_ICON_BASE_DIR, ('indicators/alert.png')) + ICON_INDI_TICK_FILE = os.path.join(HOB_ICON_BASE_DIR, ('indicators/tick.png')) + ICON_INDI_INFO_FILE = os.path.join(HOB_ICON_BASE_DIR, ('indicators/info.png')) + +class HobViewTable (gtk.VBox): + """ + A VBox to contain the table for different recipe views and package view + """ + __gsignals__ = { + "toggled" : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_PYOBJECT, + gobject.TYPE_STRING, + gobject.TYPE_INT, + gobject.TYPE_PYOBJECT,)), + "row-activated" : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_PYOBJECT, + gobject.TYPE_PYOBJECT,)), + "cell-fadeinout-stopped" : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_PYOBJECT, + gobject.TYPE_PYOBJECT, + gobject.TYPE_PYOBJECT,)), + } + + def __init__(self, columns, name): + gtk.VBox.__init__(self, False, 6) + self.table_tree = gtk.TreeView() + self.table_tree.set_headers_visible(True) + self.table_tree.set_headers_clickable(True) + self.table_tree.set_rules_hint(True) + self.table_tree.set_enable_tree_lines(True) + self.table_tree.get_selection().set_mode(gtk.SELECTION_SINGLE) + self.toggle_columns = [] + self.table_tree.connect("row-activated", self.row_activated_cb) + self.top_bar = None + self.tab_name = name + + for i, column in enumerate(columns): + col_name = column['col_name'] + col = gtk.TreeViewColumn(col_name) + col.set_clickable(True) + col.set_resizable(True) + if self.tab_name.startswith('Included'): + if col_name!='Included': + col.set_sort_column_id(column['col_id']) + else: + col.set_sort_column_id(column['col_id']) + if 'col_min' in column.keys(): + col.set_min_width(column['col_min']) + if 'col_max' in column.keys(): + col.set_max_width(column['col_max']) + if 'expand' in column.keys(): + col.set_expand(True) + self.table_tree.append_column(col) + + if (not 'col_style' in column.keys()) or column['col_style'] == 'text': + cell = gtk.CellRendererText() + col.pack_start(cell, True) + col.set_attributes(cell, text=column['col_id']) + if 'col_t_id' in column.keys(): + col.add_attribute(cell, 'font', column['col_t_id']) + elif column['col_style'] == 'check toggle': + cell = HobCellRendererToggle() + cell.set_property('activatable', True) + cell.connect("toggled", self.toggled_cb, i, self.table_tree) + cell.connect_render_state_changed(self.stop_cell_fadeinout_cb, self.table_tree) + self.toggle_id = i + col.pack_end(cell, True) + col.set_attributes(cell, active=column['col_id']) + self.toggle_columns.append(col_name) + if 'col_group' in column.keys(): + col.set_cell_data_func(cell, self.set_group_number_cb) + elif column['col_style'] == 'radio toggle': + cell = gtk.CellRendererToggle() + cell.set_property('activatable', True) + cell.set_radio(True) + cell.connect("toggled", self.toggled_cb, i, self.table_tree) + self.toggle_id = i + col.pack_end(cell, True) + col.set_attributes(cell, active=column['col_id']) + self.toggle_columns.append(col_name) + elif column['col_style'] == 'binb': + cell = gtk.CellRendererText() + col.pack_start(cell, True) + col.set_cell_data_func(cell, self.display_binb_cb, column['col_id']) + if 'col_t_id' in column.keys(): + col.add_attribute(cell, 'font', column['col_t_id']) + + self.scroll = gtk.ScrolledWindow() + self.scroll.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC) + self.scroll.add(self.table_tree) + + self.pack_end(self.scroll, True, True, 0) + + def add_no_result_bar(self, entry): + color = HobColors.KHAKI + self.top_bar = gtk.EventBox() + self.top_bar.set_size_request(-1, 70) + self.top_bar.modify_bg(gtk.STATE_NORMAL, gtk.gdk.color_parse(color)) + self.top_bar.set_flags(gtk.CAN_DEFAULT) + self.top_bar.grab_default() + + no_result_tab = gtk.Table(5, 20, True) + self.top_bar.add(no_result_tab) + + label = gtk.Label() + label.set_alignment(0.0, 0.5) + title = "No results matching your search" + label.set_markup("<span size='x-large'><b>%s</b></span>" % title) + no_result_tab.attach(label, 1, 14, 1, 4) + + clear_button = HobButton("Clear search") + clear_button.set_tooltip_text("Clear search query") + clear_button.connect('clicked', self.set_search_entry_clear_cb, entry) + no_result_tab.attach(clear_button, 16, 19, 1, 4) + + self.pack_start(self.top_bar, False, True, 12) + self.top_bar.show_all() + + def set_search_entry_clear_cb(self, button, search): + if search.get_editable() == True: + search.set_text("") + search.set_icon_sensitive(gtk.ENTRY_ICON_SECONDARY, False) + search.grab_focus() + + def display_binb_cb(self, col, cell, model, it, col_id): + binb = model.get_value(it, col_id) + # Just display the first item + if binb: + bin = binb.split(', ') + total_no = len(bin) + if total_no > 1 and bin[0] == "User Selected": + if total_no > 2: + present_binb = bin[1] + ' (+' + str(total_no - 1) + ')' + else: + present_binb = bin[1] + else: + if total_no > 1: + present_binb = bin[0] + ' (+' + str(total_no - 1) + ')' + else: + present_binb = bin[0] + cell.set_property('text', present_binb) + else: + cell.set_property('text', "") + return True + + def set_model(self, tree_model): + self.table_tree.set_model(tree_model) + + def toggle_default(self): + model = self.table_tree.get_model() + if not model: + return + iter = model.get_iter_first() + if iter: + rowpath = model.get_path(iter) + model[rowpath][self.toggle_id] = True + + def toggled_cb(self, cell, path, columnid, tree): + self.emit("toggled", cell, path, columnid, tree) + + def row_activated_cb(self, tree, path, view_column): + if not view_column.get_title() in self.toggle_columns: + self.emit("row-activated", tree.get_model(), path) + + def stop_cell_fadeinout_cb(self, ctrl, cell, tree): + self.emit("cell-fadeinout-stopped", ctrl, cell, tree) + + def set_group_number_cb(self, col, cell, model, iter): + if model and (model.iter_parent(iter) == None): + cell.cell_attr["number_of_children"] = model.iter_n_children(iter) + else: + cell.cell_attr["number_of_children"] = 0 + + def connect_group_selection(self, cb_func): + self.table_tree.get_selection().connect("changed", cb_func) + +""" +A method to calculate a softened value for the colour of widget when in the +provided state. + +widget: the widget whose style to use +state: the state of the widget to use the style for + +Returns a string value representing the softened colour +""" +def soften_color(widget, state=gtk.STATE_NORMAL): + # this colour munging routine is heavily inspired bu gdu_util_get_mix_color() + # from gnome-disk-utility: + # http://git.gnome.org/browse/gnome-disk-utility/tree/src/gdu-gtk/gdu-gtk.c?h=gnome-3-0 + blend = 0.7 + style = widget.get_style() + color = style.text[state] + color.red = color.red * blend + style.base[state].red * (1.0 - blend) + color.green = color.green * blend + style.base[state].green * (1.0 - blend) + color.blue = color.blue * blend + style.base[state].blue * (1.0 - blend) + return color.to_string() + +class BaseHobButton(gtk.Button): + """ + A gtk.Button subclass which follows the visual design of Hob for primary + action buttons + + label: the text to display as the button's label + """ + def __init__(self, label): + gtk.Button.__init__(self, label) + HobButton.style_button(self) + + @staticmethod + def style_button(button): + style = button.get_style() + style = gtk.rc_get_style_by_paths(gtk.settings_get_default(), 'gtk-button', 'gtk-button', gobject.TYPE_NONE) + + button.set_flags(gtk.CAN_DEFAULT) + button.grab_default() + +# label = "<span size='x-large'><b>%s</b></span>" % gobject.markup_escape_text(button.get_label()) + label = button.get_label() + button.set_label(label) + button.child.set_use_markup(True) + +class HobButton(BaseHobButton): + """ + A gtk.Button subclass which follows the visual design of Hob for primary + action buttons + + label: the text to display as the button's label + """ + def __init__(self, label): + BaseHobButton.__init__(self, label) + HobButton.style_button(self) + +class HobAltButton(BaseHobButton): + """ + A gtk.Button subclass which has no relief, and so is more discrete + """ + def __init__(self, label): + BaseHobButton.__init__(self, label) + HobAltButton.style_button(self) + + """ + A callback for the state-changed event to ensure the text is displayed + differently when the widget is not sensitive + """ + @staticmethod + def desensitise_on_state_change_cb(button, state): + if not button.get_property("sensitive"): + HobAltButton.set_text(button, False) + else: + HobAltButton.set_text(button, True) + + """ + Set the button label with an appropriate colour for the current widget state + """ + @staticmethod + def set_text(button, sensitive=True): + if sensitive: + colour = HobColors.PALE_BLUE + else: + colour = HobColors.LIGHT_GRAY + button.set_label("<span size='large' color='%s'><b>%s</b></span>" % (colour, gobject.markup_escape_text(button.text))) + button.child.set_use_markup(True) + +class HobImageButton(gtk.Button): + """ + A gtk.Button with an icon and two rows of text, the second of which is + displayed in a blended colour. + + primary_text: the main button label + secondary_text: optional second line of text + icon_path: path to the icon file to display on the button + """ + def __init__(self, primary_text, secondary_text="", icon_path="", hover_icon_path=""): + gtk.Button.__init__(self) + self.set_relief(gtk.RELIEF_NONE) + + self.icon_path = icon_path + self.hover_icon_path = hover_icon_path + + hbox = gtk.HBox(False, 10) + hbox.show() + self.add(hbox) + self.icon = gtk.Image() + self.icon.set_from_file(self.icon_path) + self.icon.set_alignment(0.5, 0.0) + self.icon.show() + if self.hover_icon_path and len(self.hover_icon_path): + self.connect("enter-notify-event", self.set_hover_icon_cb) + self.connect("leave-notify-event", self.set_icon_cb) + hbox.pack_start(self.icon, False, False, 0) + label = gtk.Label() + label.set_alignment(0.0, 0.5) + colour = soften_color(label) + mark = "<span size='x-large'>%s</span>\n<span size='medium' fgcolor='%s' weight='ultralight'>%s</span>" % (primary_text, colour, secondary_text) + label.set_markup(mark) + label.show() + hbox.pack_start(label, True, True, 0) + + def set_hover_icon_cb(self, widget, event): + self.icon.set_from_file(self.hover_icon_path) + + def set_icon_cb(self, widget, event): + self.icon.set_from_file(self.icon_path) + +class HobInfoButton(gtk.EventBox): + """ + This class implements a button-like widget per the Hob visual and UX designs + which will display a persistent tooltip, with the contents of tip_markup, when + clicked. + + tip_markup: the Pango Markup to be displayed in the persistent tooltip + """ + def __init__(self, tip_markup, parent=None): + gtk.EventBox.__init__(self) + self.image = gtk.Image() + self.image.set_from_file( + hic.ICON_INFO_DISPLAY_FILE) + self.image.show() + self.add(self.image) + self.tip_markup = tip_markup + self.my_parent = parent + + self.set_events(gtk.gdk.BUTTON_RELEASE | + gtk.gdk.ENTER_NOTIFY_MASK | + gtk.gdk.LEAVE_NOTIFY_MASK) + + self.connect("button-release-event", self.button_release_cb) + self.connect("enter-notify-event", self.mouse_in_cb) + self.connect("leave-notify-event", self.mouse_out_cb) + + """ + When the mouse click is released emulate a button-click and show the associated + PersistentTooltip + """ + def button_release_cb(self, widget, event): + from bb.ui.crumbs.hig.propertydialog import PropertyDialog + self.dialog = PropertyDialog(title = '', + parent = self.my_parent, + information = self.tip_markup, + flags = gtk.DIALOG_DESTROY_WITH_PARENT + | gtk.DIALOG_NO_SEPARATOR) + + button = self.dialog.add_button("Close", gtk.RESPONSE_CANCEL) + HobAltButton.style_button(button) + button.connect("clicked", lambda w: self.dialog.destroy()) + self.dialog.show_all() + self.dialog.run() + + """ + Change to the prelight image when the mouse enters the widget + """ + def mouse_in_cb(self, widget, event): + self.image.set_from_file(hic.ICON_INFO_HOVER_FILE) + + """ + Change to the stock image when the mouse enters the widget + """ + def mouse_out_cb(self, widget, event): + self.image.set_from_file(hic.ICON_INFO_DISPLAY_FILE) + +class HobIndicator(gtk.DrawingArea): + def __init__(self, count): + gtk.DrawingArea.__init__(self) + # Set no window for transparent background + self.set_has_window(False) + self.set_size_request(38,38) + # We need to pass through button clicks + self.add_events(gtk.gdk.BUTTON_PRESS_MASK | gtk.gdk.BUTTON_RELEASE_MASK) + + self.connect('expose-event', self.expose) + + self.count = count + self.color = HobColors.GRAY + + def expose(self, widget, event): + if self.count and self.count > 0: + ctx = widget.window.cairo_create() + + x, y, w, h = self.allocation + + ctx.set_operator(cairo.OPERATOR_OVER) + ctx.set_source_color(gtk.gdk.color_parse(self.color)) + ctx.translate(w/2, h/2) + ctx.arc(x, y, min(w,h)/2 - 2, 0, 2*math.pi) + ctx.fill_preserve() + + layout = self.create_pango_layout(str(self.count)) + textw, texth = layout.get_pixel_size() + x = (w/2)-(textw/2) + x + y = (h/2) - (texth/2) + y + ctx.move_to(x, y) + self.window.draw_layout(self.style.light_gc[gtk.STATE_NORMAL], int(x), int(y), layout) + + def set_count(self, count): + self.count = count + + def set_active(self, active): + if active: + self.color = HobColors.DEEP_RED + else: + self.color = HobColors.GRAY + +class HobTabLabel(gtk.HBox): + def __init__(self, text, count=0): + gtk.HBox.__init__(self, False, 0) + self.indicator = HobIndicator(count) + self.indicator.show() + self.pack_end(self.indicator, False, False) + self.lbl = gtk.Label(text) + self.lbl.set_alignment(0.0, 0.5) + self.lbl.show() + self.pack_end(self.lbl, True, True, 6) + + def set_count(self, count): + self.indicator.set_count(count) + + def set_active(self, active=True): + self.indicator.set_active(active) + +class HobNotebook(gtk.Notebook): + def __init__(self): + gtk.Notebook.__init__(self) + self.set_property('homogeneous', True) + + self.pages = [] + + self.search = None + self.search_focus = False + self.page_changed = False + + self.connect("switch-page", self.page_changed_cb) + + self.show_all() + + def page_changed_cb(self, nb, page, page_num): + for p, lbl in enumerate(self.pages): + if p == page_num: + lbl.set_active() + else: + lbl.set_active(False) + + if self.search: + self.page_changed = True + self.reset_entry(self.search, page_num) + + def append_page(self, child, tab_label, tab_tooltip=None): + label = HobTabLabel(tab_label) + if tab_tooltip: + label.set_tooltip_text(tab_tooltip) + label.set_active(False) + self.pages.append(label) + gtk.Notebook.append_page(self, child, label) + + def set_entry(self, names, tips): + self.search = gtk.Entry() + self.search_names = names + self.search_tips = tips + style = self.search.get_style() + style.text[gtk.STATE_NORMAL] = self.get_colormap().alloc_color(HobColors.GRAY, False, False) + self.search.set_style(style) + self.search.set_text(names[0]) + self.search.set_tooltip_text(self.search_tips[0]) + self.search.props.has_tooltip = True + + self.search.set_editable(False) + self.search.set_icon_from_stock(gtk.ENTRY_ICON_SECONDARY, gtk.STOCK_CLEAR) + self.search.set_icon_sensitive(gtk.ENTRY_ICON_SECONDARY, False) + self.search.connect("icon-release", self.set_search_entry_clear_cb) + self.search.set_width_chars(30) + self.search.show() + + self.search.connect("focus-in-event", self.set_search_entry_editable_cb) + self.search.connect("focus-out-event", self.set_search_entry_reset_cb) + self.set_action_widget(self.search, gtk.PACK_END) + + def show_indicator_icon(self, title, number): + for child in self.pages: + if child.lbl.get_label() == title: + child.set_count(number) + + def hide_indicator_icon(self, title): + for child in self.pages: + if child.lbl.get_label() == title: + child.set_count(0) + + def set_search_entry_editable_cb(self, search, event): + self.search_focus = True + search.set_editable(True) + text = search.get_text() + if text in self.search_names: + search.set_text("") + style = self.search.get_style() + style.text[gtk.STATE_NORMAL] = self.get_colormap().alloc_color(HobColors.BLACK, False, False) + search.set_style(style) + + def set_search_entry_reset_cb(self, search, event): + page_num = self.get_current_page() + text = search.get_text() + if not text: + self.reset_entry(search, page_num) + + def reset_entry(self, entry, page_num): + style = entry.get_style() + style.text[gtk.STATE_NORMAL] = self.get_colormap().alloc_color(HobColors.GRAY, False, False) + entry.set_style(style) + entry.set_text(self.search_names[page_num]) + entry.set_tooltip_text(self.search_tips[page_num]) + entry.set_editable(False) + entry.set_icon_sensitive(gtk.ENTRY_ICON_SECONDARY, False) + + def set_search_entry_clear_cb(self, search, icon_pos, event): + if search.get_editable() == True: + search.set_text("") + search.set_icon_sensitive(gtk.ENTRY_ICON_SECONDARY, False) + search.grab_focus() + + def set_page(self, title): + for child in self.pages: + if child.lbl.get_label() == title: + child.grab_focus() + self.set_current_page(self.pages.index(child)) + return + +class HobWarpCellRendererText(gtk.CellRendererText): + def __init__(self, col_number): + gtk.CellRendererText.__init__(self) + self.set_property("wrap-mode", pango.WRAP_WORD_CHAR) + self.set_property("wrap-width", 300) # default value wrap width is 300 + self.col_n = col_number + + def do_render(self, window, widget, background_area, cell_area, expose_area, flags): + if widget: + self.props.wrap_width = self.get_resized_wrap_width(widget, widget.get_column(self.col_n)) + return gtk.CellRendererText.do_render(self, window, widget, background_area, cell_area, expose_area, flags) + + def get_resized_wrap_width(self, treeview, column): + otherCols = [] + for col in treeview.get_columns(): + if col != column: + otherCols.append(col) + adjwidth = treeview.allocation.width - sum(c.get_width() for c in otherCols) + adjwidth -= treeview.style_get_property("horizontal-separator") * 4 + if self.props.wrap_width == adjwidth or adjwidth <= 0: + adjwidth = self.props.wrap_width + return adjwidth + +gobject.type_register(HobWarpCellRendererText) + +class HobIconChecker(hic): + def set_hob_icon_to_stock_icon(self, file_path, stock_id=""): + try: + pixbuf = gtk.gdk.pixbuf_new_from_file(file_path) + except Exception, e: + return None + + if stock_id and (gtk.icon_factory_lookup_default(stock_id) == None): + icon_factory = gtk.IconFactory() + icon_factory.add_default() + icon_factory.add(stock_id, gtk.IconSet(pixbuf)) + gtk.stock_add([(stock_id, '_label', 0, 0, '')]) + + return icon_factory.lookup(stock_id) + + return None + + """ + For make hob icon consistently by request, and avoid icon view diff by system or gtk version, we use some 'hob icon' to replace the 'gtk icon'. + this function check the stock_id and make hob_id to replaced the gtk_id then return it or "" + """ + def check_stock_icon(self, stock_name=""): + HOB_CHECK_STOCK_NAME = { + ('hic-dialog-info', 'gtk-dialog-info', 'dialog-info') : self.ICON_INDI_INFO_FILE, + ('hic-ok', 'gtk-ok', 'ok') : self.ICON_INDI_TICK_FILE, + ('hic-dialog-error', 'gtk-dialog-error', 'dialog-error') : self.ICON_INDI_ERROR_FILE, + ('hic-dialog-warning', 'gtk-dialog-warning', 'dialog-warning') : self.ICON_INDI_ALERT_FILE, + ('hic-task-refresh', 'gtk-execute', 'execute') : self.ICON_INDI_REFRESH_FILE, + } + valid_stock_id = stock_name + if stock_name: + for names, path in HOB_CHECK_STOCK_NAME.iteritems(): + if stock_name in names: + valid_stock_id = names[0] + if not gtk.icon_factory_lookup_default(valid_stock_id): + self.set_hob_icon_to_stock_icon(path, valid_stock_id) + + return valid_stock_id + +class HobCellRendererController(gobject.GObject): + (MODE_CYCLE_RUNNING, MODE_ONE_SHORT) = range(2) + __gsignals__ = { + "run-timer-stopped" : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + } + def __init__(self, runningmode=MODE_CYCLE_RUNNING, is_draw_row=False): + gobject.GObject.__init__(self) + self.timeout_id = None + self.current_angle_pos = 0.0 + self.step_angle = 0.0 + self.tree_headers_height = 0 + self.running_cell_areas = [] + self.running_mode = runningmode + self.is_queue_draw_row_area = is_draw_row + self.force_stop_enable = False + + def is_active(self): + if self.timeout_id: + return True + else: + return False + + def reset_run(self): + self.force_stop() + self.running_cell_areas = [] + self.current_angle_pos = 0.0 + self.step_angle = 0.0 + + ''' time_iterval: (1~1000)ms, which will be as the basic interval count for timer + init_usrdata: the current data which related the progress-bar will be at + min_usrdata: the range of min of user data + max_usrdata: the range of max of user data + step: each step which you want to progress + Note: the init_usrdata should in the range of from min to max, and max should > min + step should < (max - min) + ''' + def start_run(self, time_iterval, init_usrdata, min_usrdata, max_usrdata, step, tree): + if (not time_iterval) or (not max_usrdata): + return + usr_range = (max_usrdata - min_usrdata) * 1.0 + self.current_angle_pos = (init_usrdata * 1.0) / usr_range + self.step_angle = (step * 1) / usr_range + self.timeout_id = gobject.timeout_add(int(time_iterval), + self.make_image_on_progressing_cb, tree) + self.tree_headers_height = self.get_treeview_headers_height(tree) + self.force_stop_enable = False + + def force_stop(self): + self.emit("run-timer-stopped") + self.force_stop_enable = True + if self.timeout_id: + if gobject.source_remove(self.timeout_id): + self.timeout_id = None + + def on_draw_pixbuf_cb(self, pixbuf, cr, x, y, img_width, img_height, do_refresh=True): + if pixbuf: + r = max(img_width/2, img_height/2) + cr.translate(x + r, y + r) + if do_refresh: + cr.rotate(2 * math.pi * self.current_angle_pos) + + cr.set_source_pixbuf(pixbuf, -img_width/2, -img_height/2) + cr.paint() + + def on_draw_fadeinout_cb(self, cr, color, x, y, width, height, do_fadeout=True): + if do_fadeout: + alpha = self.current_angle_pos * 0.8 + else: + alpha = (1.0 - self.current_angle_pos) * 0.8 + + cr.set_source_rgba(color.red, color.green, color.blue, alpha) + cr.rectangle(x, y, width, height) + cr.fill() + + def get_treeview_headers_height(self, tree): + if tree and (tree.get_property("headers-visible") == True): + height = tree.get_allocation().height - tree.get_bin_window().get_size()[1] + return height + + return 0 + + def make_image_on_progressing_cb(self, tree): + self.current_angle_pos += self.step_angle + if self.running_mode == self.MODE_CYCLE_RUNNING: + if (self.current_angle_pos >= 1): + self.current_angle_pos = 0 + else: + if self.current_angle_pos > 1: + self.force_stop() + return False + + if self.is_queue_draw_row_area: + for path in self.running_cell_areas: + rect = tree.get_cell_area(path, tree.get_column(0)) + row_x, _, row_width, _ = tree.get_visible_rect() + tree.queue_draw_area(row_x, rect.y + self.tree_headers_height, row_width, rect.height) + else: + for rect in self.running_cell_areas: + tree.queue_draw_area(rect.x, rect.y + self.tree_headers_height, rect.width, rect.height) + + return (not self.force_stop_enable) + + def append_running_cell_area(self, cell_area): + if cell_area and (cell_area not in self.running_cell_areas): + self.running_cell_areas.append(cell_area) + + def remove_running_cell_area(self, cell_area): + if cell_area in self.running_cell_areas: + self.running_cell_areas.remove(cell_area) + if not self.running_cell_areas: + self.reset_run() + +gobject.type_register(HobCellRendererController) + +class HobCellRendererPixbuf(gtk.CellRendererPixbuf): + def __init__(self): + gtk.CellRendererPixbuf.__init__(self) + self.control = HobCellRendererController() + # add icon checker for make the gtk-icon transfer to hob-icon + self.checker = HobIconChecker() + self.set_property("stock-size", gtk.ICON_SIZE_DND) + + def get_pixbuf_from_stock_icon(self, widget, stock_id="", size=gtk.ICON_SIZE_DIALOG): + if widget and stock_id and gtk.icon_factory_lookup_default(stock_id): + return widget.render_icon(stock_id, size) + + return None + + def set_icon_name_to_id(self, new_name): + if new_name and type(new_name) == str: + # check the name is need to transfer to hob icon or not + name = self.checker.check_stock_icon(new_name) + if name.startswith("hic") or name.startswith("gtk"): + stock_id = name + else: + stock_id = 'gtk-' + name + + return stock_id + + ''' render cell exactly, "icon-name" is priority + if use the 'hic-task-refresh' will make the pix animation + if 'pix' will change the pixbuf for it from the pixbuf or image. + ''' + def do_render(self, window, tree, background_area,cell_area, expose_area, flags): + if (not self.control) or (not tree): + return + + x, y, w, h = self.on_get_size(tree, cell_area) + x += cell_area.x + y += cell_area.y + w -= 2 * self.get_property("xpad") + h -= 2 * self.get_property("ypad") + + stock_id = "" + if self.props.icon_name: + stock_id = self.set_icon_name_to_id(self.props.icon_name) + elif self.props.stock_id: + stock_id = self.props.stock_id + elif self.props.pixbuf: + pix = self.props.pixbuf + else: + return + + if stock_id: + pix = self.get_pixbuf_from_stock_icon(tree, stock_id, self.props.stock_size) + if stock_id == 'hic-task-refresh': + self.control.append_running_cell_area(cell_area) + if self.control.is_active(): + self.control.on_draw_pixbuf_cb(pix, window.cairo_create(), x, y, w, h, True) + else: + self.control.start_run(200, 0, 0, 1000, 150, tree) + else: + self.control.remove_running_cell_area(cell_area) + self.control.on_draw_pixbuf_cb(pix, window.cairo_create(), x, y, w, h, False) + + def on_get_size(self, widget, cell_area): + if self.props.icon_name or self.props.pixbuf or self.props.stock_id: + w, h = gtk.icon_size_lookup(self.props.stock_size) + calc_width = self.get_property("xpad") * 2 + w + calc_height = self.get_property("ypad") * 2 + h + x_offset = 0 + y_offset = 0 + if cell_area and w > 0 and h > 0: + x_offset = self.get_property("xalign") * (cell_area.width - calc_width - self.get_property("xpad")) + y_offset = self.get_property("yalign") * (cell_area.height - calc_height - self.get_property("ypad")) + + return x_offset, y_offset, w, h + + return 0, 0, 0, 0 + +gobject.type_register(HobCellRendererPixbuf) + +class HobCellRendererToggle(gtk.CellRendererToggle): + def __init__(self): + gtk.CellRendererToggle.__init__(self) + self.ctrl = HobCellRendererController(is_draw_row=True) + self.ctrl.running_mode = self.ctrl.MODE_ONE_SHORT + self.cell_attr = {"fadeout": False, "number_of_children": 0} + + def do_render(self, window, widget, background_area, cell_area, expose_area, flags): + if (not self.ctrl) or (not widget): + return + + if flags & gtk.CELL_RENDERER_SELECTED: + state = gtk.STATE_SELECTED + else: + state = gtk.STATE_NORMAL + + if self.ctrl.is_active(): + path = widget.get_path_at_pos(cell_area.x + cell_area.width/2, cell_area.y + cell_area.height/2) + # sometimes the parameters of cell_area will be a negative number,such as pull up down the scroll bar + # it's over the tree container range, so the path will be bad + if not path: return + path = path[0] + if path in self.ctrl.running_cell_areas: + cr = window.cairo_create() + color = widget.get_style().base[state] + + row_x, _, row_width, _ = widget.get_visible_rect() + border_y = self.get_property("ypad") + self.ctrl.on_draw_fadeinout_cb(cr, color, row_x, cell_area.y - border_y, row_width, \ + cell_area.height + border_y * 2, self.cell_attr["fadeout"]) + # draw number of a group + if self.cell_attr["number_of_children"]: + text = "%d pkg" % self.cell_attr["number_of_children"] + pangolayout = widget.create_pango_layout(text) + textw, texth = pangolayout.get_pixel_size() + x = cell_area.x + (cell_area.width/2) - (textw/2) + y = cell_area.y + (cell_area.height/2) - (texth/2) + + widget.style.paint_layout(window, state, True, cell_area, widget, "checkbox", x, y, pangolayout) + else: + return gtk.CellRendererToggle.do_render(self, window, widget, background_area, cell_area, expose_area, flags) + + '''delay: normally delay time is 1000ms + cell_list: whilch cells need to be render + ''' + def fadeout(self, tree, delay, cell_list=None): + if (delay < 200) or (not tree): + return + self.cell_attr["fadeout"] = True + self.ctrl.running_cell_areas = cell_list + self.ctrl.start_run(200, 0, 0, delay, (delay * 200 / 1000), tree) + + def connect_render_state_changed(self, func, usrdata=None): + if not func: + return + if usrdata: + self.ctrl.connect("run-timer-stopped", func, self, usrdata) + else: + self.ctrl.connect("run-timer-stopped", func, self) + +gobject.type_register(HobCellRendererToggle) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/persistenttooltip.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/persistenttooltip.py new file mode 100644 index 000000000..927c19429 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/persistenttooltip.py @@ -0,0 +1,186 @@ +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2012 Intel Corporation +# +# Authored by Joshua Lock <josh@linux.intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import gobject +import gtk +try: + import gconf +except: + pass + +class PersistentTooltip(gtk.Window): + """ + A tooltip which persists once shown until the user dismisses it with the Esc + key or by clicking the close button. + + # FIXME: the PersistentTooltip should be disabled when the user clicks anywhere off + # it. We can't do this with focus-out-event becuase modal ensures we have focus? + + markup: some Pango text markup to display in the tooltip + """ + def __init__(self, markup, parent_win=None): + gtk.Window.__init__(self, gtk.WINDOW_POPUP) + + # Inherit the system theme for a tooltip + style = gtk.rc_get_style_by_paths(gtk.settings_get_default(), + 'gtk-tooltip', 'gtk-tooltip', gobject.TYPE_NONE) + self.set_style(style) + + # The placement of the close button on the tip should reflect how the + # window manager of the users system places close buttons. Try to read + # the metacity gconf key to determine whether the close button is on the + # left or the right. + # In the case that we can't determine the users configuration we default + # to close buttons being on the right. + __button_right = True + try: + client = gconf.client_get_default() + order = client.get_string("/apps/metacity/general/button_layout") + if order and order.endswith(":"): + __button_right = False + except NameError: + pass + + # We need to ensure we're only shown once + self.shown = False + + # We don't want any WM decorations + self.set_decorated(False) + # We don't want to show in the taskbar or window switcher + self.set_skip_pager_hint(True) + self.set_skip_taskbar_hint(True) + # We must be modal to ensure we grab focus when presented from a gtk.Dialog + self.set_modal(True) + + self.set_border_width(0) + self.set_position(gtk.WIN_POS_MOUSE) + self.set_opacity(0.95) + + # Ensure a reasonable minimum size + self.set_geometry_hints(self, 100, 50) + + # Set this window as a transient window for parent(main window) + if parent_win: + self.set_transient_for(parent_win) + self.set_destroy_with_parent(True) + # Draw our label and close buttons + hbox = gtk.HBox(False, 0) + hbox.show() + self.add(hbox) + + img = gtk.Image() + img.set_from_stock(gtk.STOCK_CLOSE, gtk.ICON_SIZE_BUTTON) + + self.button = gtk.Button() + self.button.set_image(img) + self.button.connect("clicked", self._dismiss_cb) + self.button.set_flags(gtk.CAN_DEFAULT) + self.button.grab_focus() + self.button.show() + vbox = gtk.VBox(False, 0) + vbox.show() + vbox.pack_start(self.button, False, False, 0) + if __button_right: + hbox.pack_end(vbox, True, True, 0) + else: + hbox.pack_start(vbox, True, True, 0) + + self.set_default(self.button) + + bin = gtk.HBox(True, 6) + bin.set_border_width(6) + bin.show() + self.label = gtk.Label() + self.label.set_line_wrap(True) + # We want to match the colours of the normal tooltips, as dictated by + # the users gtk+-2.0 theme, wherever possible - on some systems this + # requires explicitly setting a fg_color for the label which matches the + # tooltip_fg_color + settings = gtk.settings_get_default() + colours = settings.get_property('gtk-color-scheme').split('\n') + # remove any empty lines, there's likely to be a trailing one after + # calling split on a dictionary-like string + colours = filter(None, colours) + for col in colours: + item, val = col.split(': ') + if item == 'tooltip_fg_color': + style = self.label.get_style() + style.fg[gtk.STATE_NORMAL] = gtk.gdk.color_parse(val) + self.label.set_style(style) + break # we only care for the tooltip_fg_color + + self.label.set_markup(markup) + self.label.show() + bin.add(self.label) + hbox.pack_end(bin, True, True, 6) + + # add the original URL display for user reference + if 'a href' in markup: + hbox.set_tooltip_text(self.get_markup_url(markup)) + hbox.show() + + self.connect("key-press-event", self._catch_esc_cb) + + """ + Callback when the PersistentTooltip's close button is clicked. + Hides the PersistentTooltip. + """ + def _dismiss_cb(self, button): + self.hide() + return True + + """ + Callback when the Esc key is detected. Hides the PersistentTooltip. + """ + def _catch_esc_cb(self, widget, event): + keyname = gtk.gdk.keyval_name(event.keyval) + if keyname == "Escape": + self.hide() + return True + + """ + Called to present the PersistentTooltip. + Overrides the superclasses show() method to include state tracking. + """ + def show(self): + if not self.shown: + self.shown = True + gtk.Window.show(self) + + """ + Called to hide the PersistentTooltip. + Overrides the superclasses hide() method to include state tracking. + """ + def hide(self): + self.shown = False + gtk.Window.hide(self) + + """ + Called to get the hyperlink URL from markup text. + """ + def get_markup_url(self, markup): + url = "http:" + if markup and type(markup) == str: + s = markup + if 'http:' in s: + import re + url = re.search('(http:[^,\\ "]+)', s).group(0) + + return url diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/progress.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/progress.py new file mode 100644 index 000000000..1d28a111b --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/progress.py @@ -0,0 +1,23 @@ +import gtk + +class ProgressBar(gtk.Dialog): + def __init__(self, parent): + + gtk.Dialog.__init__(self, flags=(gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT)) + self.set_title("Parsing metadata, please wait...") + self.set_default_size(500, 0) + self.set_transient_for(parent) + self.progress = gtk.ProgressBar() + self.vbox.pack_start(self.progress) + self.show_all() + + def set_text(self, msg): + self.progress.set_text(msg) + + def update(self, x, y): + self.progress.set_fraction(float(x)/float(y)) + self.progress.set_text("%2d %%" % (x*100/y)) + + def pulse(self): + self.progress.set_text("Loading...") + self.progress.pulse() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/progressbar.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/progressbar.py new file mode 100644 index 000000000..3e2c660e4 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/progressbar.py @@ -0,0 +1,59 @@ +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2011 Intel Corporation +# +# Authored by Shane Wang <shane.wang@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import gtk +from bb.ui.crumbs.hobcolor import HobColors + +class HobProgressBar (gtk.ProgressBar): + def __init__(self): + gtk.ProgressBar.__init__(self) + self.set_rcstyle(True) + self.percentage = 0 + + def set_rcstyle(self, status): + rcstyle = gtk.RcStyle() + rcstyle.fg[2] = gtk.gdk.Color(HobColors.BLACK) + if status == "stop": + rcstyle.bg[3] = gtk.gdk.Color(HobColors.WARNING) + elif status == "fail": + rcstyle.bg[3] = gtk.gdk.Color(HobColors.ERROR) + else: + rcstyle.bg[3] = gtk.gdk.Color(HobColors.RUNNING) + self.modify_style(rcstyle) + + def set_title(self, text=None): + if not text: + text = "" + text += " %.0f%%" % self.percentage + self.set_text(text) + + def set_stop_title(self, text=None): + if not text: + text = "" + self.set_text(text) + + def reset(self): + self.set_fraction(0) + self.set_text("") + self.set_rcstyle(True) + self.percentage = 0 + + def update(self, fraction): + self.percentage = int(fraction * 100) + self.set_fraction(fraction) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/puccho.glade b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/puccho.glade new file mode 100644 index 000000000..d7553a6e1 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/puccho.glade @@ -0,0 +1,606 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!DOCTYPE glade-interface SYSTEM "glade-2.0.dtd"> +<!--Generated with glade3 3.4.5 on Mon Nov 10 12:24:12 2008 --> +<glade-interface> + <widget class="GtkDialog" id="build_dialog"> + <property name="title" translatable="yes">Start a build</property> + <property name="window_position">GTK_WIN_POS_CENTER_ON_PARENT</property> + <property name="type_hint">GDK_WINDOW_TYPE_HINT_DIALOG</property> + <property name="has_separator">False</property> + <child internal-child="vbox"> + <widget class="GtkVBox" id="dialog-vbox1"> + <property name="visible">True</property> + <property name="spacing">2</property> + <child> + <widget class="GtkTable" id="build_table"> + <property name="visible">True</property> + <property name="border_width">6</property> + <property name="n_rows">7</property> + <property name="n_columns">3</property> + <property name="column_spacing">5</property> + <property name="row_spacing">6</property> + <child> + <widget class="GtkAlignment" id="status_alignment"> + <property name="visible">True</property> + <property name="left_padding">12</property> + <child> + <widget class="GtkHBox" id="status_hbox"> + <property name="spacing">6</property> + <child> + <widget class="GtkImage" id="status_image"> + <property name="visible">True</property> + <property name="no_show_all">True</property> + <property name="xalign">0</property> + <property name="stock">gtk-dialog-error</property> + </widget> + <packing> + <property name="expand">False</property> + <property name="fill">False</property> + </packing> + </child> + <child> + <widget class="GtkLabel" id="status_label"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="label" translatable="yes">If you see this text something is wrong...</property> + <property name="use_markup">True</property> + <property name="use_underline">True</property> + </widget> + <packing> + <property name="position">1</property> + </packing> + </child> + </widget> + </child> + </widget> + <packing> + <property name="right_attach">3</property> + <property name="top_attach">2</property> + <property name="bottom_attach">3</property> + </packing> + </child> + <child> + <widget class="GtkLabel" id="label2"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="label" translatable="yes"><b>Build configuration</b></property> + <property name="use_markup">True</property> + </widget> + <packing> + <property name="right_attach">3</property> + <property name="top_attach">3</property> + <property name="bottom_attach">4</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkComboBox" id="image_combo"> + <property name="visible">True</property> + <property name="sensitive">False</property> + </widget> + <packing> + <property name="left_attach">1</property> + <property name="right_attach">2</property> + <property name="top_attach">6</property> + <property name="bottom_attach">7</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkLabel" id="image_label"> + <property name="visible">True</property> + <property name="sensitive">False</property> + <property name="xalign">0</property> + <property name="xpad">12</property> + <property name="label" translatable="yes">Image:</property> + </widget> + <packing> + <property name="top_attach">6</property> + <property name="bottom_attach">7</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkComboBox" id="distribution_combo"> + <property name="visible">True</property> + <property name="sensitive">False</property> + </widget> + <packing> + <property name="left_attach">1</property> + <property name="right_attach">2</property> + <property name="top_attach">5</property> + <property name="bottom_attach">6</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkLabel" id="distribution_label"> + <property name="visible">True</property> + <property name="sensitive">False</property> + <property name="xalign">0</property> + <property name="xpad">12</property> + <property name="label" translatable="yes">Distribution:</property> + </widget> + <packing> + <property name="top_attach">5</property> + <property name="bottom_attach">6</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkComboBox" id="machine_combo"> + <property name="visible">True</property> + <property name="sensitive">False</property> + </widget> + <packing> + <property name="left_attach">1</property> + <property name="right_attach">2</property> + <property name="top_attach">4</property> + <property name="bottom_attach">5</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkLabel" id="machine_label"> + <property name="visible">True</property> + <property name="sensitive">False</property> + <property name="xalign">0</property> + <property name="xpad">12</property> + <property name="label" translatable="yes">Machine:</property> + </widget> + <packing> + <property name="top_attach">4</property> + <property name="bottom_attach">5</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkButton" id="refresh_button"> + <property name="visible">True</property> + <property name="sensitive">False</property> + <property name="can_focus">True</property> + <property name="receives_default">True</property> + <property name="label" translatable="yes">gtk-refresh</property> + <property name="use_stock">True</property> + <property name="response_id">0</property> + </widget> + <packing> + <property name="left_attach">2</property> + <property name="right_attach">3</property> + <property name="top_attach">1</property> + <property name="bottom_attach">2</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkEntry" id="location_entry"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="width_chars">32</property> + </widget> + <packing> + <property name="left_attach">1</property> + <property name="right_attach">2</property> + <property name="top_attach">1</property> + <property name="bottom_attach">2</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkLabel" id="label3"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="xpad">12</property> + <property name="label" translatable="yes">Location:</property> + </widget> + <packing> + <property name="top_attach">1</property> + <property name="bottom_attach">2</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkLabel" id="label1"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="label" translatable="yes"><b>Repository</b></property> + <property name="use_markup">True</property> + </widget> + <packing> + <property name="right_attach">3</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkAlignment" id="alignment1"> + <property name="visible">True</property> + <child> + <placeholder/> + </child> + </widget> + <packing> + <property name="left_attach">2</property> + <property name="right_attach">3</property> + <property name="top_attach">4</property> + <property name="bottom_attach">5</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkAlignment" id="alignment2"> + <property name="visible">True</property> + <child> + <placeholder/> + </child> + </widget> + <packing> + <property name="left_attach">2</property> + <property name="right_attach">3</property> + <property name="top_attach">5</property> + <property name="bottom_attach">6</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkAlignment" id="alignment3"> + <property name="visible">True</property> + <child> + <placeholder/> + </child> + </widget> + <packing> + <property name="left_attach">2</property> + <property name="right_attach">3</property> + <property name="top_attach">6</property> + <property name="bottom_attach">7</property> + <property name="y_options"></property> + </packing> + </child> + </widget> + <packing> + <property name="position">1</property> + </packing> + </child> + <child internal-child="action_area"> + <widget class="GtkHButtonBox" id="dialog-action_area1"> + <property name="visible">True</property> + <property name="layout_style">GTK_BUTTONBOX_END</property> + <child> + <placeholder/> + </child> + <child> + <placeholder/> + </child> + <child> + <placeholder/> + </child> + </widget> + <packing> + <property name="expand">False</property> + <property name="pack_type">GTK_PACK_END</property> + </packing> + </child> + </widget> + </child> + </widget> + <widget class="GtkDialog" id="dialog2"> + <property name="window_position">GTK_WIN_POS_CENTER_ON_PARENT</property> + <property name="type_hint">GDK_WINDOW_TYPE_HINT_DIALOG</property> + <property name="has_separator">False</property> + <child internal-child="vbox"> + <widget class="GtkVBox" id="dialog-vbox2"> + <property name="visible">True</property> + <property name="spacing">2</property> + <child> + <widget class="GtkTable" id="table2"> + <property name="visible">True</property> + <property name="border_width">6</property> + <property name="n_rows">7</property> + <property name="n_columns">3</property> + <property name="column_spacing">6</property> + <property name="row_spacing">6</property> + <child> + <widget class="GtkLabel" id="label7"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="label" translatable="yes"><b>Repositories</b></property> + <property name="use_markup">True</property> + </widget> + <packing> + <property name="right_attach">3</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkAlignment" id="alignment4"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="left_padding">12</property> + <child> + <widget class="GtkScrolledWindow" id="scrolledwindow1"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="hscrollbar_policy">GTK_POLICY_AUTOMATIC</property> + <property name="vscrollbar_policy">GTK_POLICY_AUTOMATIC</property> + <child> + <widget class="GtkTreeView" id="treeview1"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="headers_clickable">True</property> + </widget> + </child> + </widget> + </child> + </widget> + <packing> + <property name="right_attach">3</property> + <property name="top_attach">2</property> + <property name="bottom_attach">3</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkEntry" id="entry1"> + <property name="visible">True</property> + <property name="can_focus">True</property> + </widget> + <packing> + <property name="left_attach">1</property> + <property name="right_attach">3</property> + <property name="top_attach">1</property> + <property name="bottom_attach">2</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkLabel" id="label9"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="label" translatable="yes"><b>Additional packages</b></property> + <property name="use_markup">True</property> + </widget> + <packing> + <property name="right_attach">3</property> + <property name="top_attach">4</property> + <property name="bottom_attach">5</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkAlignment" id="alignment6"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="xscale">0</property> + <child> + <widget class="GtkLabel" id="label8"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="yalign">0</property> + <property name="xpad">12</property> + <property name="label" translatable="yes">Location: </property> + </widget> + </child> + </widget> + <packing> + <property name="top_attach">1</property> + <property name="bottom_attach">2</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkAlignment" id="alignment7"> + <property name="visible">True</property> + <property name="xalign">1</property> + <property name="xscale">0</property> + <child> + <widget class="GtkHButtonBox" id="hbuttonbox1"> + <property name="visible">True</property> + <property name="spacing">5</property> + <child> + <widget class="GtkButton" id="button7"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="receives_default">True</property> + <property name="label" translatable="yes">gtk-remove</property> + <property name="use_stock">True</property> + <property name="response_id">0</property> + </widget> + </child> + <child> + <widget class="GtkButton" id="button6"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="receives_default">True</property> + <property name="label" translatable="yes">gtk-edit</property> + <property name="use_stock">True</property> + <property name="response_id">0</property> + </widget> + <packing> + <property name="position">1</property> + </packing> + </child> + <child> + <widget class="GtkButton" id="button5"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="receives_default">True</property> + <property name="label" translatable="yes">gtk-add</property> + <property name="use_stock">True</property> + <property name="response_id">0</property> + </widget> + <packing> + <property name="position">2</property> + </packing> + </child> + </widget> + </child> + </widget> + <packing> + <property name="left_attach">1</property> + <property name="right_attach">3</property> + <property name="top_attach">3</property> + <property name="bottom_attach">4</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkAlignment" id="alignment5"> + <property name="visible">True</property> + <child> + <placeholder/> + </child> + </widget> + <packing> + <property name="top_attach">3</property> + <property name="bottom_attach">4</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkLabel" id="label10"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="yalign">0</property> + <property name="xpad">12</property> + <property name="label" translatable="yes">Search:</property> + </widget> + <packing> + <property name="top_attach">5</property> + <property name="bottom_attach">6</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkEntry" id="entry2"> + <property name="visible">True</property> + <property name="can_focus">True</property> + </widget> + <packing> + <property name="left_attach">1</property> + <property name="right_attach">3</property> + <property name="top_attach">5</property> + <property name="bottom_attach">6</property> + <property name="y_options"></property> + </packing> + </child> + <child> + <widget class="GtkAlignment" id="alignment8"> + <property name="visible">True</property> + <property name="xalign">0</property> + <property name="left_padding">12</property> + <child> + <widget class="GtkScrolledWindow" id="scrolledwindow2"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="hscrollbar_policy">GTK_POLICY_AUTOMATIC</property> + <property name="vscrollbar_policy">GTK_POLICY_AUTOMATIC</property> + <child> + <widget class="GtkTreeView" id="treeview2"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="headers_clickable">True</property> + </widget> + </child> + </widget> + </child> + </widget> + <packing> + <property name="right_attach">3</property> + <property name="top_attach">6</property> + <property name="bottom_attach">7</property> + <property name="y_options"></property> + </packing> + </child> + </widget> + <packing> + <property name="position">1</property> + </packing> + </child> + <child internal-child="action_area"> + <widget class="GtkHButtonBox" id="dialog-action_area2"> + <property name="visible">True</property> + <property name="layout_style">GTK_BUTTONBOX_END</property> + <child> + <widget class="GtkButton" id="button4"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="receives_default">True</property> + <property name="label" translatable="yes">gtk-close</property> + <property name="use_stock">True</property> + <property name="response_id">0</property> + </widget> + </child> + </widget> + <packing> + <property name="expand">False</property> + <property name="pack_type">GTK_PACK_END</property> + </packing> + </child> + </widget> + </child> + </widget> + <widget class="GtkWindow" id="main_window"> + <child> + <widget class="GtkVBox" id="main_window_vbox"> + <property name="visible">True</property> + <child> + <widget class="GtkToolbar" id="main_toolbar"> + <property name="visible">True</property> + <child> + <widget class="GtkToolButton" id="main_toolbutton_build"> + <property name="visible">True</property> + <property name="label" translatable="yes">Build</property> + <property name="stock_id">gtk-execute</property> + </widget> + <packing> + <property name="expand">False</property> + </packing> + </child> + </widget> + <packing> + <property name="expand">False</property> + </packing> + </child> + <child> + <widget class="GtkVPaned" id="vpaned1"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <child> + <widget class="GtkScrolledWindow" id="results_scrolledwindow"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="hscrollbar_policy">GTK_POLICY_AUTOMATIC</property> + <property name="vscrollbar_policy">GTK_POLICY_AUTOMATIC</property> + <child> + <placeholder/> + </child> + </widget> + <packing> + <property name="resize">False</property> + <property name="shrink">True</property> + </packing> + </child> + <child> + <widget class="GtkScrolledWindow" id="progress_scrolledwindow"> + <property name="visible">True</property> + <property name="can_focus">True</property> + <property name="hscrollbar_policy">GTK_POLICY_AUTOMATIC</property> + <property name="vscrollbar_policy">GTK_POLICY_AUTOMATIC</property> + <child> + <placeholder/> + </child> + </widget> + <packing> + <property name="resize">True</property> + <property name="shrink">True</property> + </packing> + </child> + </widget> + <packing> + <property name="position">1</property> + </packing> + </child> + </widget> + </child> + </widget> +</glade-interface> diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/runningbuild.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/runningbuild.py new file mode 100644 index 000000000..16a955d2b --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/runningbuild.py @@ -0,0 +1,551 @@ + +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2008 Intel Corporation +# +# Authored by Rob Bradford <rob@linux.intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import gtk +import gobject +import logging +import time +import urllib +import urllib2 +import pango +from bb.ui.crumbs.hobcolor import HobColors +from bb.ui.crumbs.hobwidget import HobWarpCellRendererText, HobCellRendererPixbuf + +class RunningBuildModel (gtk.TreeStore): + (COL_LOG, COL_PACKAGE, COL_TASK, COL_MESSAGE, COL_ICON, COL_COLOR, COL_NUM_ACTIVE) = range(7) + + def __init__ (self): + gtk.TreeStore.__init__ (self, + gobject.TYPE_STRING, + gobject.TYPE_STRING, + gobject.TYPE_STRING, + gobject.TYPE_STRING, + gobject.TYPE_STRING, + gobject.TYPE_STRING, + gobject.TYPE_INT) + + def failure_model_filter(self, model, it): + color = model.get(it, self.COL_COLOR)[0] + if not color: + return False + if color == HobColors.ERROR or color == HobColors.WARNING: + return True + return False + + def failure_model(self): + model = self.filter_new() + model.set_visible_func(self.failure_model_filter) + return model + + def foreach_cell_func(self, model, path, iter, usr_data=None): + if model.get_value(iter, self.COL_ICON) == "gtk-execute": + model.set(iter, self.COL_ICON, "") + + def close_task_refresh(self): + self.foreach(self.foreach_cell_func, None) + +class RunningBuild (gobject.GObject): + __gsignals__ = { + 'build-started' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'build-succeeded' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'build-failed' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'build-complete' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'build-aborted' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'task-started' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_PYOBJECT,)), + 'log-error' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'log-warning' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'disk-full' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'no-provider' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_PYOBJECT,)), + 'log' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_STRING, gobject.TYPE_PYOBJECT,)), + } + pids_to_task = {} + tasks_to_iter = {} + + def __init__ (self, sequential=False): + gobject.GObject.__init__ (self) + self.model = RunningBuildModel() + self.sequential = sequential + self.buildaborted = False + + def reset (self): + self.pids_to_task.clear() + self.tasks_to_iter.clear() + self.model.clear() + + def handle_event (self, event, pbar=None): + # Handle an event from the event queue, this may result in updating + # the model and thus the UI. Or it may be to tell us that the build + # has finished successfully (or not, as the case may be.) + + parent = None + pid = 0 + package = None + task = None + + # If we have a pid attached to this message/event try and get the + # (package, task) pair for it. If we get that then get the parent iter + # for the message. + if hasattr(event, 'pid'): + pid = event.pid + if hasattr(event, 'process'): + pid = event.process + + if pid and pid in self.pids_to_task: + (package, task) = self.pids_to_task[pid] + parent = self.tasks_to_iter[(package, task)] + + if(isinstance(event, logging.LogRecord)): + if event.taskpid == 0 or event.levelno > logging.INFO: + self.emit("log", "handle", event) + # FIXME: this is a hack! More info in Yocto #1433 + # http://bugzilla.pokylinux.org/show_bug.cgi?id=1433, temporarily + # mask the error message as it's not informative for the user. + if event.msg.startswith("Execution of event handler 'run_buildstats' failed"): + return + + if (event.levelno < logging.INFO or + event.msg.startswith("Running task")): + return # don't add these to the list + + if event.levelno >= logging.ERROR: + icon = "dialog-error" + color = HobColors.ERROR + self.emit("log-error") + elif event.levelno >= logging.WARNING: + icon = "dialog-warning" + color = HobColors.WARNING + self.emit("log-warning") + else: + icon = None + color = HobColors.OK + + # if we know which package we belong to, we'll append onto its list. + # otherwise, we'll jump to the top of the master list + if self.sequential or not parent: + tree_add = self.model.append + else: + tree_add = self.model.prepend + tree_add(parent, + (None, + package, + task, + event.getMessage(), + icon, + color, + 0)) + + # if there are warnings while processing a package + # (parent), mark the task with warning color; + # in case there are errors, the updates will be + # handled on TaskFailed. + if color == HobColors.WARNING and parent: + self.model.set(parent, self.model.COL_COLOR, color) + if task: #then we have a parent (package), and update it's color + self.model.set(self.tasks_to_iter[(package, None)], self.model.COL_COLOR, color) + + elif isinstance(event, bb.build.TaskStarted): + (package, task) = (event._package, event._task) + + # Save out this PID. + self.pids_to_task[pid] = (package, task) + + # Check if we already have this package in our model. If so then + # that can be the parent for the task. Otherwise we create a new + # top level for the package. + if ((package, None) in self.tasks_to_iter): + parent = self.tasks_to_iter[(package, None)] + else: + if self.sequential: + add = self.model.append + else: + add = self.model.prepend + parent = add(None, (None, + package, + None, + "Package: %s" % (package), + None, + HobColors.OK, + 0)) + self.tasks_to_iter[(package, None)] = parent + + # Because this parent package now has an active child mark it as + # such. + self.model.set(parent, self.model.COL_ICON, "gtk-execute") + parent_color = self.model.get(parent, self.model.COL_COLOR)[0] + if parent_color != HobColors.ERROR and parent_color != HobColors.WARNING: + self.model.set(parent, self.model.COL_COLOR, HobColors.RUNNING) + + # Add an entry in the model for this task + i = self.model.append (parent, (None, + package, + task, + "Task: %s" % (task), + "gtk-execute", + HobColors.RUNNING, + 0)) + + # update the parent's active task count + num_active = self.model.get(parent, self.model.COL_NUM_ACTIVE)[0] + 1 + self.model.set(parent, self.model.COL_NUM_ACTIVE, num_active) + + # Save out the iter so that we can find it when we have a message + # that we need to attach to a task. + self.tasks_to_iter[(package, task)] = i + + elif isinstance(event, bb.build.TaskBase): + self.emit("log", "info", event._message) + current = self.tasks_to_iter[(package, task)] + parent = self.tasks_to_iter[(package, None)] + + # remove this task from the parent's active count + num_active = self.model.get(parent, self.model.COL_NUM_ACTIVE)[0] - 1 + self.model.set(parent, self.model.COL_NUM_ACTIVE, num_active) + + if isinstance(event, bb.build.TaskFailed): + # Mark the task and parent as failed + icon = "dialog-error" + color = HobColors.ERROR + + logfile = event.logfile + if logfile and os.path.exists(logfile): + with open(logfile) as f: + logdata = f.read() + self.model.append(current, ('pastebin', None, None, logdata, 'gtk-error', HobColors.OK, 0)) + + for i in (current, parent): + self.model.set(i, self.model.COL_ICON, icon, + self.model.COL_COLOR, color) + else: + # Mark the parent package and the task as inactive, + # but make sure to preserve error, warnings and active + # states + parent_color = self.model.get(parent, self.model.COL_COLOR)[0] + task_color = self.model.get(current, self.model.COL_COLOR)[0] + + # Mark the task as inactive + self.model.set(current, self.model.COL_ICON, None) + if task_color != HobColors.ERROR: + if task_color == HobColors.WARNING: + self.model.set(current, self.model.COL_ICON, 'dialog-warning') + else: + self.model.set(current, self.model.COL_COLOR, HobColors.OK) + + # Mark the parent as inactive + if parent_color != HobColors.ERROR: + if parent_color == HobColors.WARNING: + self.model.set(parent, self.model.COL_ICON, "dialog-warning") + else: + self.model.set(parent, self.model.COL_ICON, None) + if num_active == 0: + self.model.set(parent, self.model.COL_COLOR, HobColors.OK) + + # Clear the iters and the pids since when the task goes away the + # pid will no longer be used for messages + del self.tasks_to_iter[(package, task)] + del self.pids_to_task[pid] + + elif isinstance(event, bb.event.BuildStarted): + + self.emit("build-started") + self.model.prepend(None, (None, + None, + None, + "Build Started (%s)" % time.strftime('%m/%d/%Y %H:%M:%S'), + None, + HobColors.OK, + 0)) + if pbar: + pbar.update(0, self.progress_total) + pbar.set_title(bb.event.getName(event)) + + elif isinstance(event, bb.event.BuildCompleted): + failures = int (event._failures) + self.model.prepend(None, (None, + None, + None, + "Build Completed (%s)" % time.strftime('%m/%d/%Y %H:%M:%S'), + None, + HobColors.OK, + 0)) + + # Emit the appropriate signal depending on the number of failures + if self.buildaborted: + self.emit ("build-aborted") + self.buildaborted = False + elif (failures >= 1): + self.emit ("build-failed") + else: + self.emit ("build-succeeded") + # Emit a generic "build-complete" signal for things wishing to + # handle when the build is finished + self.emit("build-complete") + # reset the all cell's icon indicator + self.model.close_task_refresh() + if pbar: + pbar.set_text(event.msg) + + elif isinstance(event, bb.event.DiskFull): + self.buildaborted = True + self.emit("disk-full") + + elif isinstance(event, bb.command.CommandFailed): + self.emit("log", "error", "Command execution failed: %s" % (event.error)) + if event.error.startswith("Exited with"): + # If the command fails with an exit code we're done, emit the + # generic signal for the UI to notify the user + self.emit("build-complete") + # reset the all cell's icon indicator + self.model.close_task_refresh() + + elif isinstance(event, bb.event.CacheLoadStarted) and pbar: + pbar.set_title("Loading cache") + self.progress_total = event.total + pbar.update(0, self.progress_total) + elif isinstance(event, bb.event.CacheLoadProgress) and pbar: + pbar.update(event.current, self.progress_total) + elif isinstance(event, bb.event.CacheLoadCompleted) and pbar: + pbar.update(self.progress_total, self.progress_total) + pbar.hide() + elif isinstance(event, bb.event.ParseStarted) and pbar: + if event.total == 0: + return + pbar.set_title("Processing recipes") + self.progress_total = event.total + pbar.update(0, self.progress_total) + elif isinstance(event, bb.event.ParseProgress) and pbar: + pbar.update(event.current, self.progress_total) + elif isinstance(event, bb.event.ParseCompleted) and pbar: + pbar.hide() + #using runqueue events as many as possible to update the progress bar + elif isinstance(event, bb.runqueue.runQueueTaskFailed): + self.emit("log", "error", "Task %s (%s) failed with exit code '%s'" % (event.taskid, event.taskstring, event.exitcode)) + elif isinstance(event, bb.runqueue.sceneQueueTaskFailed): + self.emit("log", "warn", "Setscene task %s (%s) failed with exit code '%s' - real task will be run instead" \ + % (event.taskid, event.taskstring, event.exitcode)) + elif isinstance(event, (bb.runqueue.runQueueTaskStarted, bb.runqueue.sceneQueueTaskStarted)): + if isinstance(event, bb.runqueue.sceneQueueTaskStarted): + self.emit("log", "info", "Running setscene task %d of %d (%s)" % \ + (event.stats.completed + event.stats.active + event.stats.failed + 1, + event.stats.total, event.taskstring)) + else: + if event.noexec: + tasktype = 'noexec task' + else: + tasktype = 'task' + self.emit("log", "info", "Running %s %s of %s (ID: %s, %s)" % \ + (tasktype, event.stats.completed + event.stats.active + event.stats.failed + 1, + event.stats.total, event.taskid, event.taskstring)) + message = {} + message["eventname"] = bb.event.getName(event) + num_of_completed = event.stats.completed + event.stats.failed + message["current"] = num_of_completed + message["total"] = event.stats.total + message["title"] = "" + message["task"] = event.taskstring + self.emit("task-started", message) + elif isinstance(event, bb.event.MultipleProviders): + self.emit("log", "info", "multiple providers are available for %s%s (%s)" \ + % (event._is_runtime and "runtime " or "", event._item, ", ".join(event._candidates))) + self.emit("log", "info", "consider defining a PREFERRED_PROVIDER entry to match %s" % (event._item)) + elif isinstance(event, bb.event.NoProvider): + msg = "" + if event._runtime: + r = "R" + else: + r = "" + + extra = '' + if not event._reasons: + if event._close_matches: + extra = ". Close matches:\n %s" % '\n '.join(event._close_matches) + + if event._dependees: + msg = "Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s\n" % (r, event._item, ", ".join(event._dependees), r, extra) + else: + msg = "Nothing %sPROVIDES '%s'%s\n" % (r, event._item, extra) + if event._reasons: + for reason in event._reasons: + msg += ("%s\n" % reason) + self.emit("no-provider", msg) + self.emit("log", "error", msg) + elif isinstance(event, bb.event.LogExecTTY): + icon = "dialog-warning" + color = HobColors.WARNING + if self.sequential or not parent: + tree_add = self.model.append + else: + tree_add = self.model.prepend + tree_add(parent, + (None, + package, + task, + event.msg, + icon, + color, + 0)) + else: + if not isinstance(event, (bb.event.BuildBase, + bb.event.StampUpdate, + bb.event.ConfigParsed, + bb.event.RecipeParsed, + bb.event.RecipePreFinalise, + bb.runqueue.runQueueEvent, + bb.runqueue.runQueueExitWait, + bb.event.OperationStarted, + bb.event.OperationCompleted, + bb.event.OperationProgress)): + self.emit("log", "error", "Unknown event: %s" % (event.error if hasattr(event, 'error') else 'error')) + + return + + +def do_pastebin(text): + url = 'http://pastebin.com/api_public.php' + params = {'paste_code': text, 'paste_format': 'text'} + + req = urllib2.Request(url, urllib.urlencode(params)) + response = urllib2.urlopen(req) + paste_url = response.read() + + return paste_url + + +class RunningBuildTreeView (gtk.TreeView): + __gsignals__ = { + "button_press_event" : "override" + } + def __init__ (self, readonly=False, hob=False): + gtk.TreeView.__init__ (self) + self.readonly = readonly + + # The icon that indicates whether we're building or failed. + # add 'hob' flag because there has not only hob to share this code + if hob: + renderer = HobCellRendererPixbuf () + else: + renderer = gtk.CellRendererPixbuf() + col = gtk.TreeViewColumn ("Status", renderer) + col.add_attribute (renderer, "icon-name", 4) + self.append_column (col) + + # The message of the build. + # add 'hob' flag because there has not only hob to share this code + if hob: + self.message_renderer = HobWarpCellRendererText (col_number=1) + else: + self.message_renderer = gtk.CellRendererText () + self.message_column = gtk.TreeViewColumn ("Message", self.message_renderer, text=3) + self.message_column.add_attribute(self.message_renderer, 'background', 5) + self.message_renderer.set_property('editable', (not self.readonly)) + self.append_column (self.message_column) + + def do_button_press_event(self, event): + gtk.TreeView.do_button_press_event(self, event) + + if event.button == 3: + selection = super(RunningBuildTreeView, self).get_selection() + (model, it) = selection.get_selected() + if it is not None: + can_paste = model.get(it, model.COL_LOG)[0] + if can_paste == 'pastebin': + # build a simple menu with a pastebin option + menu = gtk.Menu() + menuitem = gtk.MenuItem("Copy") + menu.append(menuitem) + menuitem.connect("activate", self.clipboard_handler, (model, it)) + menuitem.show() + menuitem = gtk.MenuItem("Send log to pastebin") + menu.append(menuitem) + menuitem.connect("activate", self.pastebin_handler, (model, it)) + menuitem.show() + menu.show() + menu.popup(None, None, None, event.button, event.time) + + def _add_to_clipboard(self, clipping): + """ + Add the contents of clipping to the system clipboard. + """ + clipboard = gtk.clipboard_get() + clipboard.set_text(clipping) + clipboard.store() + + def pastebin_handler(self, widget, data): + """ + Send the log data to pastebin, then add the new paste url to the + clipboard. + """ + (model, it) = data + paste_url = do_pastebin(model.get(it, model.COL_MESSAGE)[0]) + + # @todo Provide visual feedback to the user that it is done and that + # it worked. + print paste_url + + self._add_to_clipboard(paste_url) + + def clipboard_handler(self, widget, data): + """ + """ + (model, it) = data + message = model.get(it, model.COL_MESSAGE)[0] + + self._add_to_clipboard(message) + +class BuildFailureTreeView(gtk.TreeView): + + def __init__ (self): + gtk.TreeView.__init__(self) + self.set_rules_hint(False) + self.set_headers_visible(False) + self.get_selection().set_mode(gtk.SELECTION_SINGLE) + + # The icon that indicates whether we're building or failed. + renderer = HobCellRendererPixbuf () + col = gtk.TreeViewColumn ("Status", renderer) + col.add_attribute (renderer, "icon-name", RunningBuildModel.COL_ICON) + self.append_column (col) + + # The message of the build. + self.message_renderer = HobWarpCellRendererText (col_number=1) + self.message_column = gtk.TreeViewColumn ("Message", self.message_renderer, text=RunningBuildModel.COL_MESSAGE, background=RunningBuildModel.COL_COLOR) + self.append_column (self.message_column) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/utils.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/utils.py new file mode 100644 index 000000000..939864fa6 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/crumbs/utils.py @@ -0,0 +1,34 @@ +# +# BitBake UI Utils +# +# Copyright (C) 2012 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This utility method looks for xterm or vte and return the +# frist to exist, currently we are keeping this simple, but +# we will likely move the oe.terminal implementation into +# bitbake which will allow more flexibility. + +import os +import bb + +def which_terminal(): + term = bb.utils.which(os.environ["PATH"], "xterm") + if term: + return term + " -e " + term = bb.utils.which(os.environ["PATH"], "vte") + if term: + return term + " -c " + return None diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/depexp.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/depexp.py new file mode 100644 index 000000000..240aafc3e --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/depexp.py @@ -0,0 +1,333 @@ +# +# BitBake Graphical GTK based Dependency Explorer +# +# Copyright (C) 2007 Ross Burton +# Copyright (C) 2007 - 2008 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import gobject +import gtk +import Queue +import threading +import xmlrpclib +import bb +import bb.event +from bb.ui.crumbs.progressbar import HobProgressBar + +# Package Model +(COL_PKG_NAME) = (0) + +# Dependency Model +(TYPE_DEP, TYPE_RDEP) = (0, 1) +(COL_DEP_TYPE, COL_DEP_PARENT, COL_DEP_PACKAGE) = (0, 1, 2) + + +class PackageDepView(gtk.TreeView): + def __init__(self, model, dep_type, label): + gtk.TreeView.__init__(self) + self.current = None + self.dep_type = dep_type + self.filter_model = model.filter_new() + self.filter_model.set_visible_func(self._filter) + self.set_model(self.filter_model) + #self.connect("row-activated", self.on_package_activated, COL_DEP_PACKAGE) + self.append_column(gtk.TreeViewColumn(label, gtk.CellRendererText(), text=COL_DEP_PACKAGE)) + + def _filter(self, model, iter): + (this_type, package) = model.get(iter, COL_DEP_TYPE, COL_DEP_PARENT) + if this_type != self.dep_type: return False + return package == self.current + + def set_current_package(self, package): + self.current = package + self.filter_model.refilter() + + +class PackageReverseDepView(gtk.TreeView): + def __init__(self, model, label): + gtk.TreeView.__init__(self) + self.current = None + self.filter_model = model.filter_new() + self.filter_model.set_visible_func(self._filter) + self.set_model(self.filter_model) + self.append_column(gtk.TreeViewColumn(label, gtk.CellRendererText(), text=COL_DEP_PARENT)) + + def _filter(self, model, iter): + package = model.get_value(iter, COL_DEP_PACKAGE) + return package == self.current + + def set_current_package(self, package): + self.current = package + self.filter_model.refilter() + + +class DepExplorer(gtk.Window): + def __init__(self): + gtk.Window.__init__(self) + self.set_title("Dependency Explorer") + self.set_default_size(500, 500) + self.connect("delete-event", gtk.main_quit) + + # Create the data models + self.pkg_model = gtk.ListStore(gobject.TYPE_STRING) + self.pkg_model.set_sort_column_id(COL_PKG_NAME, gtk.SORT_ASCENDING) + self.depends_model = gtk.ListStore(gobject.TYPE_INT, gobject.TYPE_STRING, gobject.TYPE_STRING) + self.depends_model.set_sort_column_id(COL_DEP_PACKAGE, gtk.SORT_ASCENDING) + + pane = gtk.HPaned() + pane.set_position(250) + self.add(pane) + + # The master list of packages + scrolled = gtk.ScrolledWindow() + scrolled.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) + scrolled.set_shadow_type(gtk.SHADOW_IN) + + self.pkg_treeview = gtk.TreeView(self.pkg_model) + self.pkg_treeview.get_selection().connect("changed", self.on_cursor_changed) + column = gtk.TreeViewColumn("Package", gtk.CellRendererText(), text=COL_PKG_NAME) + self.pkg_treeview.append_column(column) + pane.add1(scrolled) + scrolled.add(self.pkg_treeview) + + box = gtk.VBox(homogeneous=True, spacing=4) + + # Runtime Depends + scrolled = gtk.ScrolledWindow() + scrolled.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) + scrolled.set_shadow_type(gtk.SHADOW_IN) + self.rdep_treeview = PackageDepView(self.depends_model, TYPE_RDEP, "Runtime Depends") + self.rdep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PACKAGE) + scrolled.add(self.rdep_treeview) + box.add(scrolled) + + # Build Depends + scrolled = gtk.ScrolledWindow() + scrolled.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) + scrolled.set_shadow_type(gtk.SHADOW_IN) + self.dep_treeview = PackageDepView(self.depends_model, TYPE_DEP, "Build Depends") + self.dep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PACKAGE) + scrolled.add(self.dep_treeview) + box.add(scrolled) + pane.add2(box) + + # Reverse Depends + scrolled = gtk.ScrolledWindow() + scrolled.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) + scrolled.set_shadow_type(gtk.SHADOW_IN) + self.revdep_treeview = PackageReverseDepView(self.depends_model, "Reverse Depends") + self.revdep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PARENT) + scrolled.add(self.revdep_treeview) + box.add(scrolled) + pane.add2(box) + + self.show_all() + + def on_package_activated(self, treeview, path, column, data_col): + model = treeview.get_model() + package = model.get_value(model.get_iter(path), data_col) + + pkg_path = [] + def finder(model, path, iter, needle): + package = model.get_value(iter, COL_PKG_NAME) + if package == needle: + pkg_path.append(path) + return True + else: + return False + self.pkg_model.foreach(finder, package) + if pkg_path: + self.pkg_treeview.get_selection().select_path(pkg_path[0]) + self.pkg_treeview.scroll_to_cell(pkg_path[0]) + + def on_cursor_changed(self, selection): + (model, it) = selection.get_selected() + if it is None: + current_package = None + else: + current_package = model.get_value(it, COL_PKG_NAME) + self.rdep_treeview.set_current_package(current_package) + self.dep_treeview.set_current_package(current_package) + self.revdep_treeview.set_current_package(current_package) + + + def parse(self, depgraph): + for package in depgraph["pn"]: + self.pkg_model.insert(0, (package,)) + + for package in depgraph["depends"]: + for depend in depgraph["depends"][package]: + self.depends_model.insert (0, (TYPE_DEP, package, depend)) + + for package in depgraph["rdepends-pn"]: + for rdepend in depgraph["rdepends-pn"][package]: + self.depends_model.insert (0, (TYPE_RDEP, package, rdepend)) + + +class gtkthread(threading.Thread): + quit = threading.Event() + def __init__(self, shutdown): + threading.Thread.__init__(self) + self.setDaemon(True) + self.shutdown = shutdown + + def run(self): + gobject.threads_init() + gtk.gdk.threads_init() + gtk.main() + gtkthread.quit.set() + + +def main(server, eventHandler, params): + try: + params.updateFromServer(server) + cmdline = params.parseActions() + if not cmdline: + print("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + return 1 + if 'msg' in cmdline and cmdline['msg']: + print(cmdline['msg']) + return 1 + cmdline = cmdline['action'] + if not cmdline or cmdline[0] != "generateDotGraph": + print("This UI requires the -g option") + return 1 + ret, error = server.runCommand(["generateDepTreeEvent", cmdline[1], cmdline[2]]) + if error: + print("Error running command '%s': %s" % (cmdline, error)) + return 1 + elif ret != True: + print("Error running command '%s': returned %s" % (cmdline, ret)) + return 1 + except xmlrpclib.Fault as x: + print("XMLRPC Fault getting commandline:\n %s" % x) + return + + try: + gtk.init_check() + except RuntimeError: + sys.stderr.write("Please set DISPLAY variable before running this command \n") + return + + shutdown = 0 + + gtkgui = gtkthread(shutdown) + gtkgui.start() + + gtk.gdk.threads_enter() + dep = DepExplorer() + bardialog = gtk.Dialog(parent=dep, + flags=gtk.DIALOG_MODAL|gtk.DIALOG_DESTROY_WITH_PARENT) + bardialog.set_default_size(400, 50) + pbar = HobProgressBar() + bardialog.vbox.pack_start(pbar) + bardialog.show_all() + bardialog.connect("delete-event", gtk.main_quit) + gtk.gdk.threads_leave() + + progress_total = 0 + while True: + try: + event = eventHandler.waitEvent(0.25) + if gtkthread.quit.isSet(): + _, error = server.runCommand(["stateForceShutdown"]) + if error: + print('Unable to cleanly stop: %s' % error) + break + + if event is None: + continue + + if isinstance(event, bb.event.CacheLoadStarted): + progress_total = event.total + gtk.gdk.threads_enter() + bardialog.set_title("Loading Cache") + pbar.update(0) + gtk.gdk.threads_leave() + + if isinstance(event, bb.event.CacheLoadProgress): + x = event.current + gtk.gdk.threads_enter() + pbar.update(x * 1.0 / progress_total) + pbar.set_title('') + gtk.gdk.threads_leave() + continue + + if isinstance(event, bb.event.CacheLoadCompleted): + bardialog.hide() + continue + + if isinstance(event, bb.event.ParseStarted): + progress_total = event.total + if progress_total == 0: + continue + gtk.gdk.threads_enter() + pbar.update(0) + bardialog.set_title("Processing recipes") + + gtk.gdk.threads_leave() + + if isinstance(event, bb.event.ParseProgress): + x = event.current + gtk.gdk.threads_enter() + pbar.update(x * 1.0 / progress_total) + pbar.set_title('') + gtk.gdk.threads_leave() + continue + + if isinstance(event, bb.event.ParseCompleted): + bardialog.hide() + continue + + if isinstance(event, bb.event.DepTreeGenerated): + gtk.gdk.threads_enter() + dep.parse(event._depgraph) + gtk.gdk.threads_leave() + + if isinstance(event, bb.command.CommandCompleted): + continue + + if isinstance(event, bb.command.CommandFailed): + print("Command execution failed: %s" % event.error) + return event.exitcode + + if isinstance(event, bb.command.CommandExit): + return event.exitcode + + if isinstance(event, bb.cooker.CookerExit): + break + + continue + except EnvironmentError as ioerror: + # ignore interrupted io + if ioerror.args[0] == 4: + pass + except KeyboardInterrupt: + if shutdown == 2: + print("\nThird Keyboard Interrupt, exit.\n") + break + if shutdown == 1: + print("\nSecond Keyboard Interrupt, stopping...\n") + _, error = server.runCommand(["stateForceShutdown"]) + if error: + print('Unable to cleanly stop: %s' % error) + if shutdown == 0: + print("\nKeyboard Interrupt, closing down...\n") + _, error = server.runCommand(["stateShutdown"]) + if error: + print('Unable to cleanly shutdown: %s' % error) + shutdown = shutdown + 1 + pass diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/goggle.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/goggle.py new file mode 100644 index 000000000..f4ee7b41a --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/goggle.py @@ -0,0 +1,121 @@ +# +# BitBake Graphical GTK User Interface +# +# Copyright (C) 2008 Intel Corporation +# +# Authored by Rob Bradford <rob@linux.intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import gobject +import gtk +import xmlrpclib +from bb.ui.crumbs.runningbuild import RunningBuildTreeView, RunningBuild +from bb.ui.crumbs.progress import ProgressBar + +import Queue + + +def event_handle_idle_func (eventHandler, build, pbar): + + # Consume as many messages as we can in the time available to us + event = eventHandler.getEvent() + while event: + build.handle_event (event, pbar) + event = eventHandler.getEvent() + + return True + +def scroll_tv_cb (model, path, iter, view): + view.scroll_to_cell (path) + + +# @todo hook these into the GUI so the user has feedback... +def running_build_failed_cb (running_build): + pass + + +def running_build_succeeded_cb (running_build): + pass + + +class MainWindow (gtk.Window): + def __init__ (self): + gtk.Window.__init__ (self, gtk.WINDOW_TOPLEVEL) + + # Setup tree view and the scrolled window + scrolled_window = gtk.ScrolledWindow () + self.add (scrolled_window) + self.cur_build_tv = RunningBuildTreeView() + self.connect("delete-event", gtk.main_quit) + self.set_default_size(640, 480) + scrolled_window.add (self.cur_build_tv) + + +def main (server, eventHandler, params): + gobject.threads_init() + gtk.gdk.threads_init() + + window = MainWindow () + window.show_all () + pbar = ProgressBar(window) + pbar.connect("delete-event", gtk.main_quit) + + # Create the object for the current build + running_build = RunningBuild () + window.cur_build_tv.set_model (running_build.model) + running_build.model.connect("row-inserted", scroll_tv_cb, window.cur_build_tv) + running_build.connect ("build-succeeded", running_build_succeeded_cb) + running_build.connect ("build-failed", running_build_failed_cb) + + try: + params.updateFromServer(server) + cmdline = params.parseActions() + if not cmdline: + print("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + return 1 + if 'msg' in cmdline and cmdline['msg']: + logger.error(cmdline['msg']) + return 1 + cmdline = cmdline['action'] + ret, error = server.runCommand(cmdline) + if error: + print("Error running command '%s': %s" % (cmdline, error)) + return 1 + elif ret != True: + print("Error running command '%s': returned %s" % (cmdline, ret)) + return 1 + except xmlrpclib.Fault as x: + print("XMLRPC Fault getting commandline:\n %s" % x) + return 1 + + # Use a timeout function for probing the event queue to find out if we + # have a message waiting for us. + gobject.timeout_add (100, + event_handle_idle_func, + eventHandler, + running_build, + pbar) + + try: + gtk.main() + except EnvironmentError as ioerror: + # ignore interrupted io + if ioerror.args[0] == 4: + pass + except KeyboardInterrupt: + pass + finally: + server.runCommand(["stateForceShutdown"]) + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/images/images_display.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/images/images_display.png new file mode 100644 index 000000000..a7f87101a Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/images/images_display.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/images/images_hover.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/images/images_hover.png new file mode 100644 index 000000000..2d9cd99b8 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/images/images_hover.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/add-hover.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/add-hover.png new file mode 100644 index 000000000..526df770d Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/add-hover.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/add.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/add.png new file mode 100644 index 000000000..31e7090d6 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/add.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/alert.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/alert.png new file mode 100644 index 000000000..d1c6f55a2 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/alert.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/confirmation.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/confirmation.png new file mode 100644 index 000000000..3a5402d1e Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/confirmation.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/denied.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/denied.png new file mode 100644 index 000000000..ee35c7def Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/denied.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/error.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/error.png new file mode 100644 index 000000000..d06a8c151 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/error.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/info.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/info.png new file mode 100644 index 000000000..ee8e8d846 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/info.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/issues.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/issues.png new file mode 100644 index 000000000..b0c746133 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/issues.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/refresh.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/refresh.png new file mode 100644 index 000000000..eb6c419db Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/refresh.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/remove-hover.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/remove-hover.png new file mode 100644 index 000000000..aa57c6998 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/remove-hover.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/remove.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/remove.png new file mode 100644 index 000000000..05c3c293d Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/remove.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/tick.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/tick.png new file mode 100644 index 000000000..beaad361c Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/indicators/tick.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/info/info_display.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/info/info_display.png new file mode 100644 index 000000000..5afbba29f Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/info/info_display.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/info/info_hover.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/info/info_hover.png new file mode 100644 index 000000000..f9d294dfa Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/info/info_hover.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/layers/layers_display.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/layers/layers_display.png new file mode 100644 index 000000000..b7f9053a9 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/layers/layers_display.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/layers/layers_hover.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/layers/layers_hover.png new file mode 100644 index 000000000..0bf3ce0db Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/layers/layers_hover.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/packages/packages_display.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/packages/packages_display.png new file mode 100644 index 000000000..f5d0a5064 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/packages/packages_display.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/packages/packages_hover.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/packages/packages_hover.png new file mode 100644 index 000000000..c081165f3 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/packages/packages_hover.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/recipe/recipe_display.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/recipe/recipe_display.png new file mode 100644 index 000000000..e9809bc7d Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/recipe/recipe_display.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/recipe/recipe_hover.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/recipe/recipe_hover.png new file mode 100644 index 000000000..7e48da9af Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/recipe/recipe_hover.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/settings/settings_display.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/settings/settings_display.png new file mode 100644 index 000000000..88c464db0 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/settings/settings_display.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/settings/settings_hover.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/settings/settings_hover.png new file mode 100644 index 000000000..d92a0bf2c Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/settings/settings_hover.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/templates/templates_display.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/templates/templates_display.png new file mode 100644 index 000000000..153c7afb6 Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/templates/templates_display.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/templates/templates_hover.png b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/templates/templates_hover.png new file mode 100644 index 000000000..afb7165fe Binary files /dev/null and b/import-layers/yocto-poky/bitbake/lib/bb/ui/icons/templates/templates_hover.png differ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/knotty.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/knotty.py new file mode 100644 index 000000000..268562770 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/knotty.py @@ -0,0 +1,594 @@ +# +# BitBake (No)TTY UI Implementation +# +# Handling output to TTYs or files (no TTY) +# +# Copyright (C) 2006-2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import division + +import os +import sys +import xmlrpclib +import logging +import progressbar +import signal +import bb.msg +import time +import fcntl +import struct +import copy +import atexit +from bb.ui import uihelper + +featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS] + +logger = logging.getLogger("BitBake") +interactive = sys.stdout.isatty() + +class BBProgress(progressbar.ProgressBar): + def __init__(self, msg, maxval): + self.msg = msg + widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ', + progressbar.ETA()] + + try: + self._resize_default = signal.getsignal(signal.SIGWINCH) + except: + self._resize_default = None + progressbar.ProgressBar.__init__(self, maxval, [self.msg + ": "] + widgets, fd=sys.stdout) + + def _handle_resize(self, signum, frame): + progressbar.ProgressBar._handle_resize(self, signum, frame) + if self._resize_default: + self._resize_default(signum, frame) + def finish(self): + progressbar.ProgressBar.finish(self) + if self._resize_default: + signal.signal(signal.SIGWINCH, self._resize_default) + +class NonInteractiveProgress(object): + fobj = sys.stdout + + def __init__(self, msg, maxval): + self.msg = msg + self.maxval = maxval + + def start(self): + self.fobj.write("%s..." % self.msg) + self.fobj.flush() + return self + + def update(self, value): + pass + + def finish(self): + self.fobj.write("done.\n") + self.fobj.flush() + +def new_progress(msg, maxval): + if interactive: + return BBProgress(msg, maxval) + else: + return NonInteractiveProgress(msg, maxval) + +def pluralise(singular, plural, qty): + if(qty == 1): + return singular % qty + else: + return plural % qty + + +class InteractConsoleLogFilter(logging.Filter): + def __init__(self, tf, format): + self.tf = tf + self.format = format + + def filter(self, record): + if record.levelno == self.format.NOTE and (record.msg.startswith("Running") or record.msg.startswith("recipe ")): + return False + self.tf.clearFooter() + return True + +class TerminalFilter(object): + rows = 25 + columns = 80 + + def sigwinch_handle(self, signum, frame): + self.rows, self.columns = self.getTerminalColumns() + if self._sigwinch_default: + self._sigwinch_default(signum, frame) + + def getTerminalColumns(self): + def ioctl_GWINSZ(fd): + try: + cr = struct.unpack('hh', fcntl.ioctl(fd, self.termios.TIOCGWINSZ, '1234')) + except: + return None + return cr + cr = ioctl_GWINSZ(sys.stdout.fileno()) + if not cr: + try: + fd = os.open(os.ctermid(), os.O_RDONLY) + cr = ioctl_GWINSZ(fd) + os.close(fd) + except: + pass + if not cr: + try: + cr = (env['LINES'], env['COLUMNS']) + except: + cr = (25, 80) + return cr + + def __init__(self, main, helper, console, errconsole, format): + self.main = main + self.helper = helper + self.cuu = None + self.stdinbackup = None + self.interactive = sys.stdout.isatty() + self.footer_present = False + self.lastpids = [] + + if not self.interactive: + return + + try: + import curses + except ImportError: + sys.exit("FATAL: The knotty ui could not load the required curses python module.") + + import termios + self.curses = curses + self.termios = termios + try: + fd = sys.stdin.fileno() + self.stdinbackup = termios.tcgetattr(fd) + new = copy.deepcopy(self.stdinbackup) + new[3] = new[3] & ~termios.ECHO + termios.tcsetattr(fd, termios.TCSADRAIN, new) + curses.setupterm() + if curses.tigetnum("colors") > 2: + format.enable_color() + self.ed = curses.tigetstr("ed") + if self.ed: + self.cuu = curses.tigetstr("cuu") + try: + self._sigwinch_default = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, self.sigwinch_handle) + except: + pass + self.rows, self.columns = self.getTerminalColumns() + except: + self.cuu = None + if not self.cuu: + self.interactive = False + bb.note("Unable to use interactive mode for this terminal, using fallback") + return + console.addFilter(InteractConsoleLogFilter(self, format)) + errconsole.addFilter(InteractConsoleLogFilter(self, format)) + + def clearFooter(self): + if self.footer_present: + lines = self.footer_present + sys.stdout.write(self.curses.tparm(self.cuu, lines)) + sys.stdout.write(self.curses.tparm(self.ed)) + self.footer_present = False + + def updateFooter(self): + if not self.cuu: + return + activetasks = self.helper.running_tasks + failedtasks = self.helper.failed_tasks + runningpids = self.helper.running_pids + if self.footer_present and (self.lastcount == self.helper.tasknumber_current) and (self.lastpids == runningpids): + return + if self.footer_present: + self.clearFooter() + if (not self.helper.tasknumber_total or self.helper.tasknumber_current == self.helper.tasknumber_total) and not len(activetasks): + return + tasks = [] + for t in runningpids: + tasks.append("%s (pid %s)" % (activetasks[t]["title"], t)) + + if self.main.shutdown: + content = "Waiting for %s running tasks to finish:" % len(activetasks) + elif not len(activetasks): + content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total) + else: + content = "Currently %s running tasks (%s of %s):" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total) + print(content) + lines = 1 + int(len(content) / (self.columns + 1)) + for tasknum, task in enumerate(tasks[:(self.rows - 2)]): + content = "%s: %s" % (tasknum, task) + print(content) + lines = lines + 1 + int(len(content) / (self.columns + 1)) + self.footer_present = lines + self.lastpids = runningpids[:] + self.lastcount = self.helper.tasknumber_current + + def finish(self): + if self.stdinbackup: + fd = sys.stdin.fileno() + self.termios.tcsetattr(fd, self.termios.TCSADRAIN, self.stdinbackup) + +def _log_settings_from_server(server): + # Get values of variables which control our output + includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"]) + if error: + logger.error("Unable to get the value of BBINCLUDELOGS variable: %s" % error) + raise BaseException(error) + loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"]) + if error: + logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error) + raise BaseException(error) + consolelogfile, error = server.runCommand(["getSetVariable", "BB_CONSOLELOG"]) + if error: + logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" % error) + raise BaseException(error) + return includelogs, loglines, consolelogfile + +_evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.LogRecord", + "bb.build.TaskFailed", "bb.build.TaskBase", "bb.event.ParseStarted", + "bb.event.ParseProgress", "bb.event.ParseCompleted", "bb.event.CacheLoadStarted", + "bb.event.CacheLoadProgress", "bb.event.CacheLoadCompleted", "bb.command.CommandFailed", + "bb.command.CommandExit", "bb.command.CommandCompleted", "bb.cooker.CookerExit", + "bb.event.MultipleProviders", "bb.event.NoProvider", "bb.runqueue.sceneQueueTaskStarted", + "bb.runqueue.runQueueTaskStarted", "bb.runqueue.runQueueTaskFailed", "bb.runqueue.sceneQueueTaskFailed", + "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent"] + +def main(server, eventHandler, params, tf = TerminalFilter): + + includelogs, loglines, consolelogfile = _log_settings_from_server(server) + + if sys.stdin.isatty() and sys.stdout.isatty(): + log_exec_tty = True + else: + log_exec_tty = False + + helper = uihelper.BBUIHelper() + + console = logging.StreamHandler(sys.stdout) + errconsole = logging.StreamHandler(sys.stderr) + format_str = "%(levelname)s: %(message)s" + format = bb.msg.BBLogFormatter(format_str) + bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut) + bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr) + console.setFormatter(format) + errconsole.setFormatter(format) + logger.addHandler(console) + logger.addHandler(errconsole) + + bb.utils.set_process_name("KnottyUI") + + if params.options.remote_server and params.options.kill_server: + server.terminateServer() + return + + if consolelogfile and not params.options.show_environment and not params.options.show_versions: + bb.utils.mkdirhier(os.path.dirname(consolelogfile)) + conlogformat = bb.msg.BBLogFormatter(format_str) + consolelog = logging.FileHandler(consolelogfile) + bb.msg.addDefaultlogFilter(consolelog) + consolelog.setFormatter(conlogformat) + logger.addHandler(consolelog) + + llevel, debug_domains = bb.msg.constructLogOptions() + server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list]) + + universe = False + if not params.observe_only: + params.updateFromServer(server) + params.updateToServer(server, os.environ.copy()) + cmdline = params.parseActions() + if not cmdline: + print("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + return 1 + if 'msg' in cmdline and cmdline['msg']: + logger.error(cmdline['msg']) + return 1 + if cmdline['action'][0] == "buildTargets" and "universe" in cmdline['action'][1]: + universe = True + + ret, error = server.runCommand(cmdline['action']) + if error: + logger.error("Command '%s' failed: %s" % (cmdline, error)) + return 1 + elif ret != True: + logger.error("Command '%s' failed: returned %s" % (cmdline, ret)) + return 1 + + + parseprogress = None + cacheprogress = None + main.shutdown = 0 + interrupted = False + return_value = 0 + errors = 0 + warnings = 0 + taskfailures = [] + + termfilter = tf(main, helper, console, errconsole, format) + atexit.register(termfilter.finish) + + while True: + try: + event = eventHandler.waitEvent(0) + if event is None: + if main.shutdown > 1: + break + termfilter.updateFooter() + event = eventHandler.waitEvent(0.25) + if event is None: + continue + helper.eventHandler(event) + if isinstance(event, bb.runqueue.runQueueExitWait): + if not main.shutdown: + main.shutdown = 1 + continue + if isinstance(event, bb.event.LogExecTTY): + if log_exec_tty: + tries = event.retries + while tries: + print("Trying to run: %s" % event.prog) + if os.system(event.prog) == 0: + break + time.sleep(event.sleep_delay) + tries -= 1 + if tries: + continue + logger.warn(event.msg) + continue + + if isinstance(event, logging.LogRecord): + if event.levelno >= format.ERROR: + errors = errors + 1 + return_value = 1 + elif event.levelno == format.WARNING: + warnings = warnings + 1 + + if event.taskpid != 0: + # For "normal" logging conditions, don't show note logs from tasks + # but do show them if the user has changed the default log level to + # include verbose/debug messages + if event.levelno <= format.NOTE and (event.levelno < llevel or (event.levelno == format.NOTE and llevel != format.VERBOSE)): + continue + + # Prefix task messages with recipe/task + if event.taskpid in helper.running_tasks: + taskinfo = helper.running_tasks[event.taskpid] + event.msg = taskinfo['title'] + ': ' + event.msg + if hasattr(event, 'fn'): + event.msg = event.fn + ': ' + event.msg + logger.handle(event) + continue + + if isinstance(event, bb.build.TaskFailedSilent): + logger.warn("Logfile for failed setscene task is %s" % event.logfile) + continue + if isinstance(event, bb.build.TaskFailed): + return_value = 1 + logfile = event.logfile + if logfile and os.path.exists(logfile): + termfilter.clearFooter() + bb.error("Logfile of failure stored in: %s" % logfile) + if includelogs and not event.errprinted: + print("Log data follows:") + f = open(logfile, "r") + lines = [] + while True: + l = f.readline() + if l == '': + break + l = l.rstrip() + if loglines: + lines.append(' | %s' % l) + if len(lines) > int(loglines): + lines.pop(0) + else: + print('| %s' % l) + f.close() + if lines: + for line in lines: + print(line) + if isinstance(event, bb.build.TaskBase): + logger.info(event._message) + continue + if isinstance(event, bb.event.ParseStarted): + if event.total == 0: + continue + parseprogress = new_progress("Parsing recipes", event.total).start() + continue + if isinstance(event, bb.event.ParseProgress): + parseprogress.update(event.current) + continue + if isinstance(event, bb.event.ParseCompleted): + if not parseprogress: + continue + + parseprogress.finish() + print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors." + % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors))) + continue + + if isinstance(event, bb.event.CacheLoadStarted): + cacheprogress = new_progress("Loading cache", event.total).start() + continue + if isinstance(event, bb.event.CacheLoadProgress): + cacheprogress.update(event.current) + continue + if isinstance(event, bb.event.CacheLoadCompleted): + cacheprogress.finish() + print("Loaded %d entries from dependency cache." % event.num_entries) + continue + + if isinstance(event, bb.command.CommandFailed): + return_value = event.exitcode + if event.error: + errors = errors + 1 + logger.error("Command execution failed: %s", event.error) + main.shutdown = 2 + continue + if isinstance(event, bb.command.CommandExit): + if not return_value: + return_value = event.exitcode + continue + if isinstance(event, (bb.command.CommandCompleted, bb.cooker.CookerExit)): + main.shutdown = 2 + continue + if isinstance(event, bb.event.MultipleProviders): + logger.info("multiple providers are available for %s%s (%s)", event._is_runtime and "runtime " or "", + event._item, + ", ".join(event._candidates)) + rtime = "" + if event._is_runtime: + rtime = "R" + logger.info("consider defining a PREFERRED_%sPROVIDER entry to match %s" % (rtime, event._item)) + continue + if isinstance(event, bb.event.NoProvider): + if event._runtime: + r = "R" + else: + r = "" + + extra = '' + if not event._reasons: + if event._close_matches: + extra = ". Close matches:\n %s" % '\n '.join(event._close_matches) + + # For universe builds, only show these as warnings, not errors + h = logger.warning + if not universe: + return_value = 1 + errors = errors + 1 + h = logger.error + + if event._dependees: + h("Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s", r, event._item, ", ".join(event._dependees), r, extra) + else: + h("Nothing %sPROVIDES '%s'%s", r, event._item, extra) + if event._reasons: + for reason in event._reasons: + h("%s", reason) + continue + + if isinstance(event, bb.runqueue.sceneQueueTaskStarted): + logger.info("Running setscene task %d of %d (%s)" % (event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total, event.taskstring)) + continue + + if isinstance(event, bb.runqueue.runQueueTaskStarted): + if event.noexec: + tasktype = 'noexec task' + else: + tasktype = 'task' + logger.info("Running %s %s of %s (ID: %s, %s)", + tasktype, + event.stats.completed + event.stats.active + + event.stats.failed + 1, + event.stats.total, event.taskid, event.taskstring) + continue + + if isinstance(event, bb.runqueue.runQueueTaskFailed): + return_value = 1 + taskfailures.append(event.taskstring) + logger.error("Task %s (%s) failed with exit code '%s'", + event.taskid, event.taskstring, event.exitcode) + continue + + if isinstance(event, bb.runqueue.sceneQueueTaskFailed): + logger.warn("Setscene task %s (%s) failed with exit code '%s' - real task will be run instead", + event.taskid, event.taskstring, event.exitcode) + continue + + if isinstance(event, bb.event.DepTreeGenerated): + continue + + # ignore + if isinstance(event, (bb.event.BuildBase, + bb.event.MetadataEvent, + bb.event.StampUpdate, + bb.event.ConfigParsed, + bb.event.RecipeParsed, + bb.event.RecipePreFinalise, + bb.runqueue.runQueueEvent, + bb.event.OperationStarted, + bb.event.OperationCompleted, + bb.event.OperationProgress, + bb.event.DiskFull)): + continue + + logger.error("Unknown event: %s", event) + + except EnvironmentError as ioerror: + termfilter.clearFooter() + # ignore interrupted io + if ioerror.args[0] == 4: + continue + sys.stderr.write(str(ioerror)) + if not params.observe_only: + _, error = server.runCommand(["stateForceShutdown"]) + main.shutdown = 2 + except KeyboardInterrupt: + termfilter.clearFooter() + if params.observe_only: + print("\nKeyboard Interrupt, exiting observer...") + main.shutdown = 2 + if not params.observe_only and main.shutdown == 1: + print("\nSecond Keyboard Interrupt, stopping...\n") + _, error = server.runCommand(["stateForceShutdown"]) + if error: + logger.error("Unable to cleanly stop: %s" % error) + if not params.observe_only and main.shutdown == 0: + print("\nKeyboard Interrupt, closing down...\n") + interrupted = True + _, error = server.runCommand(["stateShutdown"]) + if error: + logger.error("Unable to cleanly shutdown: %s" % error) + main.shutdown = main.shutdown + 1 + pass + except Exception as e: + import traceback + sys.stderr.write(traceback.format_exc()) + if not params.observe_only: + _, error = server.runCommand(["stateForceShutdown"]) + main.shutdown = 2 + return_value = 1 + try: + summary = "" + if taskfailures: + summary += pluralise("\nSummary: %s task failed:", + "\nSummary: %s tasks failed:", len(taskfailures)) + for failure in taskfailures: + summary += "\n %s" % failure + if warnings: + summary += pluralise("\nSummary: There was %s WARNING message shown.", + "\nSummary: There were %s WARNING messages shown.", warnings) + if return_value and errors: + summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.", + "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors) + if summary: + print(summary) + + if interrupted: + print("Execution was interrupted, returning a non-zero exit code.") + if return_value == 0: + return_value = 1 + except IOError as e: + import errno + if e.errno == errno.EPIPE: + pass + + return return_value diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/ncurses.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/ncurses.py new file mode 100644 index 000000000..9589a77d7 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/ncurses.py @@ -0,0 +1,373 @@ +# +# BitBake Curses UI Implementation +# +# Implements an ncurses frontend for the BitBake utility. +# +# Copyright (C) 2006 Michael 'Mickey' Lauer +# Copyright (C) 2006-2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + We have the following windows: + + 1.) Main Window: Shows what we are ultimately building and how far we are. Includes status bar + 2.) Thread Activity Window: Shows one status line for every concurrent bitbake thread. + 3.) Command Line Window: Contains an interactive command line where you can interact w/ Bitbake. + + Basic window layout is like that: + + |---------------------------------------------------------| + | <Main Window> | <Thread Activity Window> | + | | 0: foo do_compile complete| + | Building Gtk+-2.6.10 | 1: bar do_patch complete | + | Status: 60% | ... | + | | ... | + | | ... | + |---------------------------------------------------------| + |<Command Line Window> | + |>>> which virtual/kernel | + |openzaurus-kernel | + |>>> _ | + |---------------------------------------------------------| + +""" + + +from __future__ import division +import logging +import os, sys, itertools, time, subprocess + +try: + import curses +except ImportError: + sys.exit("FATAL: The ncurses ui could not load the required curses python module.") + +import bb +import xmlrpclib +from bb import ui +from bb.ui import uihelper + +parsespin = itertools.cycle( r'|/-\\' ) + +X = 0 +Y = 1 +WIDTH = 2 +HEIGHT = 3 + +MAXSTATUSLENGTH = 32 + +class NCursesUI: + """ + NCurses UI Class + """ + class Window: + """Base Window Class""" + def __init__( self, x, y, width, height, fg=curses.COLOR_BLACK, bg=curses.COLOR_WHITE ): + self.win = curses.newwin( height, width, y, x ) + self.dimensions = ( x, y, width, height ) + """ + if curses.has_colors(): + color = 1 + curses.init_pair( color, fg, bg ) + self.win.bkgdset( ord(' '), curses.color_pair(color) ) + else: + self.win.bkgdset( ord(' '), curses.A_BOLD ) + """ + self.erase() + self.setScrolling() + self.win.noutrefresh() + + def erase( self ): + self.win.erase() + + def setScrolling( self, b = True ): + self.win.scrollok( b ) + self.win.idlok( b ) + + def setBoxed( self ): + self.boxed = True + self.win.box() + self.win.noutrefresh() + + def setText( self, x, y, text, *args ): + self.win.addstr( y, x, text, *args ) + self.win.noutrefresh() + + def appendText( self, text, *args ): + self.win.addstr( text, *args ) + self.win.noutrefresh() + + def drawHline( self, y ): + self.win.hline( y, 0, curses.ACS_HLINE, self.dimensions[WIDTH] ) + self.win.noutrefresh() + + class DecoratedWindow( Window ): + """Base class for windows with a box and a title bar""" + def __init__( self, title, x, y, width, height, fg=curses.COLOR_BLACK, bg=curses.COLOR_WHITE ): + NCursesUI.Window.__init__( self, x+1, y+3, width-2, height-4, fg, bg ) + self.decoration = NCursesUI.Window( x, y, width, height, fg, bg ) + self.decoration.setBoxed() + self.decoration.win.hline( 2, 1, curses.ACS_HLINE, width-2 ) + self.setTitle( title ) + + def setTitle( self, title ): + self.decoration.setText( 1, 1, title.center( self.dimensions[WIDTH]-2 ), curses.A_BOLD ) + + #-------------------------------------------------------------------------# +# class TitleWindow( Window ): + #-------------------------------------------------------------------------# +# """Title Window""" +# def __init__( self, x, y, width, height ): +# NCursesUI.Window.__init__( self, x, y, width, height ) +# version = bb.__version__ +# title = "BitBake %s" % version +# credit = "(C) 2003-2007 Team BitBake" +# #self.win.hline( 2, 1, curses.ACS_HLINE, width-2 ) +# self.win.border() +# self.setText( 1, 1, title.center( self.dimensions[WIDTH]-2 ), curses.A_BOLD ) +# self.setText( 1, 2, credit.center( self.dimensions[WIDTH]-2 ), curses.A_BOLD ) + + #-------------------------------------------------------------------------# + class ThreadActivityWindow( DecoratedWindow ): + #-------------------------------------------------------------------------# + """Thread Activity Window""" + def __init__( self, x, y, width, height ): + NCursesUI.DecoratedWindow.__init__( self, "Thread Activity", x, y, width, height ) + + def setStatus( self, thread, text ): + line = "%02d: %s" % ( thread, text ) + width = self.dimensions[WIDTH] + if ( len(line) > width ): + line = line[:width-3] + "..." + else: + line = line.ljust( width ) + self.setText( 0, thread, line ) + + #-------------------------------------------------------------------------# + class MainWindow( DecoratedWindow ): + #-------------------------------------------------------------------------# + """Main Window""" + def __init__( self, x, y, width, height ): + self.StatusPosition = width - MAXSTATUSLENGTH + NCursesUI.DecoratedWindow.__init__( self, None, x, y, width, height ) + curses.nl() + + def setTitle( self, title ): + title = "BitBake %s" % bb.__version__ + self.decoration.setText( 2, 1, title, curses.A_BOLD ) + self.decoration.setText( self.StatusPosition - 8, 1, "Status:", curses.A_BOLD ) + + def setStatus(self, status): + while len(status) < MAXSTATUSLENGTH: + status = status + " " + self.decoration.setText( self.StatusPosition, 1, status, curses.A_BOLD ) + + + #-------------------------------------------------------------------------# + class ShellOutputWindow( DecoratedWindow ): + #-------------------------------------------------------------------------# + """Interactive Command Line Output""" + def __init__( self, x, y, width, height ): + NCursesUI.DecoratedWindow.__init__( self, "Command Line Window", x, y, width, height ) + + #-------------------------------------------------------------------------# + class ShellInputWindow( Window ): + #-------------------------------------------------------------------------# + """Interactive Command Line Input""" + def __init__( self, x, y, width, height ): + NCursesUI.Window.__init__( self, x, y, width, height ) + +# put that to the top again from curses.textpad import Textbox +# self.textbox = Textbox( self.win ) +# t = threading.Thread() +# t.run = self.textbox.edit +# t.start() + + #-------------------------------------------------------------------------# + def main(self, stdscr, server, eventHandler, params): + #-------------------------------------------------------------------------# + height, width = stdscr.getmaxyx() + + # for now split it like that: + # MAIN_y + THREAD_y = 2/3 screen at the top + # MAIN_x = 2/3 left, THREAD_y = 1/3 right + # CLI_y = 1/3 of screen at the bottom + # CLI_x = full + + main_left = 0 + main_top = 0 + main_height = ( height // 3 * 2 ) + main_width = ( width // 3 ) * 2 + clo_left = main_left + clo_top = main_top + main_height + clo_height = height - main_height - main_top - 1 + clo_width = width + cli_left = main_left + cli_top = clo_top + clo_height + cli_height = 1 + cli_width = width + thread_left = main_left + main_width + thread_top = main_top + thread_height = main_height + thread_width = width - main_width + + #tw = self.TitleWindow( 0, 0, width, main_top ) + mw = self.MainWindow( main_left, main_top, main_width, main_height ) + taw = self.ThreadActivityWindow( thread_left, thread_top, thread_width, thread_height ) + clo = self.ShellOutputWindow( clo_left, clo_top, clo_width, clo_height ) + cli = self.ShellInputWindow( cli_left, cli_top, cli_width, cli_height ) + cli.setText( 0, 0, "BB>" ) + + mw.setStatus("Idle") + + helper = uihelper.BBUIHelper() + shutdown = 0 + + try: + params.updateFromServer(server) + cmdline = params.parseActions() + if not cmdline: + print("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + return 1 + if 'msg' in cmdline and cmdline['msg']: + logger.error(cmdline['msg']) + return 1 + cmdline = cmdline['action'] + ret, error = server.runCommand(cmdline) + if error: + print("Error running command '%s': %s" % (cmdline, error)) + return + elif ret != True: + print("Couldn't get default commandlind! %s" % ret) + return + except xmlrpclib.Fault as x: + print("XMLRPC Fault getting commandline:\n %s" % x) + return + + exitflag = False + while not exitflag: + try: + event = eventHandler.waitEvent(0.25) + if not event: + continue + + helper.eventHandler(event) + if isinstance(event, bb.build.TaskBase): + mw.appendText("NOTE: %s\n" % event._message) + if isinstance(event, logging.LogRecord): + mw.appendText(logging.getLevelName(event.levelno) + ': ' + event.getMessage() + '\n') + + if isinstance(event, bb.event.CacheLoadStarted): + self.parse_total = event.total + if isinstance(event, bb.event.CacheLoadProgress): + x = event.current + y = self.parse_total + mw.setStatus("Loading Cache: %s [%2d %%]" % ( next(parsespin), x*100/y ) ) + if isinstance(event, bb.event.CacheLoadCompleted): + mw.setStatus("Idle") + mw.appendText("Loaded %d entries from dependency cache.\n" + % ( event.num_entries)) + + if isinstance(event, bb.event.ParseStarted): + self.parse_total = event.total + if isinstance(event, bb.event.ParseProgress): + x = event.current + y = self.parse_total + mw.setStatus("Parsing Recipes: %s [%2d %%]" % ( next(parsespin), x*100/y ) ) + if isinstance(event, bb.event.ParseCompleted): + mw.setStatus("Idle") + mw.appendText("Parsing finished. %d cached, %d parsed, %d skipped, %d masked.\n" + % ( event.cached, event.parsed, event.skipped, event.masked )) + +# if isinstance(event, bb.build.TaskFailed): +# if event.logfile: +# if data.getVar("BBINCLUDELOGS", d): +# bb.error("log data follows (%s)" % logfile) +# number_of_lines = data.getVar("BBINCLUDELOGS_LINES", d) +# if number_of_lines: +# subprocess.call('tail -n%s %s' % (number_of_lines, logfile), shell=True) +# else: +# f = open(logfile, "r") +# while True: +# l = f.readline() +# if l == '': +# break +# l = l.rstrip() +# print '| %s' % l +# f.close() +# else: +# bb.error("see log in %s" % logfile) + + if isinstance(event, bb.command.CommandCompleted): + # stop so the user can see the result of the build, but + # also allow them to now exit with a single ^C + shutdown = 2 + if isinstance(event, bb.command.CommandFailed): + mw.appendText("Command execution failed: %s" % event.error) + time.sleep(2) + exitflag = True + if isinstance(event, bb.command.CommandExit): + exitflag = True + if isinstance(event, bb.cooker.CookerExit): + exitflag = True + + if isinstance(event, bb.event.LogExecTTY): + mw.appendText('WARN: ' + event.msg + '\n') + if helper.needUpdate: + activetasks, failedtasks = helper.getTasks() + taw.erase() + taw.setText(0, 0, "") + if activetasks: + taw.appendText("Active Tasks:\n") + for task in activetasks.itervalues(): + taw.appendText(task["title"] + '\n') + if failedtasks: + taw.appendText("Failed Tasks:\n") + for task in failedtasks: + taw.appendText(task["title"] + '\n') + + curses.doupdate() + except EnvironmentError as ioerror: + # ignore interrupted io + if ioerror.args[0] == 4: + pass + + except KeyboardInterrupt: + if shutdown == 2: + mw.appendText("Third Keyboard Interrupt, exit.\n") + exitflag = True + if shutdown == 1: + mw.appendText("Second Keyboard Interrupt, stopping...\n") + _, error = server.runCommand(["stateForceShutdown"]) + if error: + print("Unable to cleanly stop: %s" % error) + if shutdown == 0: + mw.appendText("Keyboard Interrupt, closing down...\n") + _, error = server.runCommand(["stateShutdown"]) + if error: + print("Unable to cleanly shutdown: %s" % error) + shutdown = shutdown + 1 + pass + +def main(server, eventHandler, params): + if not os.isatty(sys.stdout.fileno()): + print("FATAL: Unable to run 'ncurses' UI without a TTY.") + return + ui = NCursesUI() + try: + curses.wrapper(ui.main, server, eventHandler, params) + except: + import traceback + traceback.print_exc() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/toasterui.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/toasterui.py new file mode 100644 index 000000000..6bf4c1f03 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/toasterui.py @@ -0,0 +1,465 @@ +# +# BitBake ToasterUI Implementation +# based on (No)TTY UI Implementation by Richard Purdie +# +# Handling output to TTYs or files (no TTY) +# +# Copyright (C) 2006-2012 Richard Purdie +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import division +import time +import sys +try: + import bb +except RuntimeError as exc: + sys.exit(str(exc)) + +from bb.ui import uihelper +from bb.ui.buildinfohelper import BuildInfoHelper + +import bb.msg +import logging +import os + +# pylint: disable=invalid-name +# module properties for UI modules are read by bitbake and the contract should not be broken + + +featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING, bb.cooker.CookerFeatures.SEND_SANITYEVENTS] + +logger = logging.getLogger("ToasterLogger") +interactive = sys.stdout.isatty() + +def _log_settings_from_server(server): + # Get values of variables which control our output + includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"]) + if error: + logger.error("Unable to get the value of BBINCLUDELOGS variable: %s", error) + raise BaseException(error) + loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"]) + if error: + logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s", error) + raise BaseException(error) + consolelogfile, error = server.runCommand(["getVariable", "BB_CONSOLELOG"]) + if error: + logger.error("Unable to get the value of BB_CONSOLELOG variable: %s", error) + raise BaseException(error) + return consolelogfile + +# create a log file for a single build and direct the logger at it; +# log file name is timestamped to the millisecond (depending +# on system clock accuracy) to ensure it doesn't overlap with +# other log file names +# +# returns (log file, path to log file) for a build +def _open_build_log(log_dir): + format_str = "%(levelname)s: %(message)s" + + now = time.time() + now_ms = int((now - int(now)) * 1000) + time_str = time.strftime('build_%Y%m%d_%H%M%S', time.localtime(now)) + log_file_name = time_str + ('.%d.log' % now_ms) + build_log_file_path = os.path.join(log_dir, log_file_name) + + build_log = logging.FileHandler(build_log_file_path) + + logformat = bb.msg.BBLogFormatter(format_str) + build_log.setFormatter(logformat) + + bb.msg.addDefaultlogFilter(build_log) + logger.addHandler(build_log) + + return (build_log, build_log_file_path) + +# stop logging to the build log if it exists +def _close_build_log(build_log): + if build_log: + build_log.flush() + build_log.close() + logger.removeHandler(build_log) + +_evt_list = [ + "bb.build.TaskBase", + "bb.build.TaskFailed", + "bb.build.TaskFailedSilent", + "bb.build.TaskStarted", + "bb.build.TaskSucceeded", + "bb.command.CommandCompleted", + "bb.command.CommandExit", + "bb.command.CommandFailed", + "bb.cooker.CookerExit", + "bb.event.BuildCompleted", + "bb.event.BuildStarted", + "bb.event.CacheLoadCompleted", + "bb.event.CacheLoadProgress", + "bb.event.CacheLoadStarted", + "bb.event.ConfigParsed", + "bb.event.DepTreeGenerated", + "bb.event.LogExecTTY", + "bb.event.MetadataEvent", + "bb.event.MultipleProviders", + "bb.event.NoProvider", + "bb.event.ParseCompleted", + "bb.event.ParseProgress", + "bb.event.RecipeParsed", + "bb.event.SanityCheck", + "bb.event.SanityCheckPassed", + "bb.event.TreeDataPreparationCompleted", + "bb.event.TreeDataPreparationStarted", + "bb.runqueue.runQueueTaskCompleted", + "bb.runqueue.runQueueTaskFailed", + "bb.runqueue.runQueueTaskSkipped", + "bb.runqueue.runQueueTaskStarted", + "bb.runqueue.sceneQueueTaskCompleted", + "bb.runqueue.sceneQueueTaskFailed", + "bb.runqueue.sceneQueueTaskStarted", + "logging.LogRecord"] + +def main(server, eventHandler, params): + # set to a logging.FileHandler instance when a build starts; + # see _open_build_log() + build_log = None + + # set to the log path when a build starts + build_log_file_path = None + + helper = uihelper.BBUIHelper() + + # TODO don't use log output to determine when bitbake has started + # + # WARNING: this log handler cannot be removed, as localhostbecontroller + # relies on output in the toaster_ui.log file to determine whether + # the bitbake server has started, which only happens if + # this logger is setup here (see the TODO in the loop below) + console = logging.StreamHandler(sys.stdout) + format_str = "%(levelname)s: %(message)s" + formatter = bb.msg.BBLogFormatter(format_str) + bb.msg.addDefaultlogFilter(console) + console.setFormatter(formatter) + logger.addHandler(console) + logger.setLevel(logging.INFO) + llevel, debug_domains = bb.msg.constructLogOptions() + result, error = server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list]) + if not result or error: + logger.error("can't set event mask: %s", error) + return 1 + + # verify and warn + build_history_enabled = True + inheritlist, _ = server.runCommand(["getVariable", "INHERIT"]) + + if not "buildhistory" in inheritlist.split(" "): + logger.warn("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.") + build_history_enabled = False + + if not params.observe_only: + params.updateFromServer(server) + params.updateToServer(server, os.environ.copy()) + cmdline = params.parseActions() + if not cmdline: + print("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + return 1 + if 'msg' in cmdline and cmdline['msg']: + logger.error(cmdline['msg']) + return 1 + + ret, error = server.runCommand(cmdline['action']) + if error: + logger.error("Command '%s' failed: %s" % (cmdline, error)) + return 1 + elif ret != True: + logger.error("Command '%s' failed: returned %s" % (cmdline, ret)) + return 1 + + # set to 1 when toasterui needs to shut down + main.shutdown = 0 + + interrupted = False + return_value = 0 + errors = 0 + warnings = 0 + taskfailures = [] + first = True + + buildinfohelper = BuildInfoHelper(server, build_history_enabled, + os.getenv('TOASTER_BRBE')) + + # write our own log files into bitbake's log directory; + # we're only interested in the path to the parent directory of + # this file, as we're writing our own logs into the same directory + consolelogfile = _log_settings_from_server(server) + log_dir = os.path.dirname(consolelogfile) + bb.utils.mkdirhier(log_dir) + + while True: + try: + event = eventHandler.waitEvent(0.25) + if first: + first = False + + # TODO don't use log output to determine when bitbake has started + # + # this is the line localhostbecontroller needs to + # see in toaster_ui.log which it uses to decide whether + # the bitbake server has started... + logger.info("ToasterUI waiting for events") + + if event is None: + if main.shutdown > 0: + # if shutting down, close any open build log first + _close_build_log(build_log) + + break + continue + + helper.eventHandler(event) + + # pylint: disable=protected-access + # the code will look into the protected variables of the event; no easy way around this + + # we treat ParseStarted as the first event of toaster-triggered + # builds; that way we get the Build Configuration included in the log + # and any errors that occur before BuildStarted is fired + if isinstance(event, bb.event.ParseStarted): + if not (build_log and build_log_file_path): + build_log, build_log_file_path = _open_build_log(log_dir) + continue + + if isinstance(event, bb.event.BuildStarted): + if not (build_log and build_log_file_path): + build_log, build_log_file_path = _open_build_log(log_dir) + + buildinfohelper.store_started_build(event, build_log_file_path) + continue + + if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): + buildinfohelper.update_and_store_task(event) + logger.info("Logfile for task %s", event.logfile) + continue + + if isinstance(event, bb.build.TaskBase): + logger.info(event._message) + + if isinstance(event, bb.event.LogExecTTY): + logger.info(event.msg) + continue + + if isinstance(event, logging.LogRecord): + if event.levelno == -1: + event.levelno = formatter.ERROR + + buildinfohelper.store_log_event(event) + + if event.levelno >= formatter.ERROR: + errors = errors + 1 + elif event.levelno == formatter.WARNING: + warnings = warnings + 1 + + # For "normal" logging conditions, don't show note logs from tasks + # but do show them if the user has changed the default log level to + # include verbose/debug messages + if event.taskpid != 0 and event.levelno <= formatter.NOTE: + continue + + logger.handle(event) + continue + + if isinstance(event, bb.build.TaskFailed): + buildinfohelper.update_and_store_task(event) + logfile = event.logfile + if logfile and os.path.exists(logfile): + bb.error("Logfile of failure stored in: %s" % logfile) + continue + + # these events are unprocessed now, but may be used in the future to log + # timing and error informations from the parsing phase in Toaster + if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)): + continue + if isinstance(event, bb.event.ParseProgress): + continue + if isinstance(event, bb.event.ParseCompleted): + continue + if isinstance(event, bb.event.CacheLoadStarted): + continue + if isinstance(event, bb.event.CacheLoadProgress): + continue + if isinstance(event, bb.event.CacheLoadCompleted): + continue + if isinstance(event, bb.event.MultipleProviders): + logger.info("multiple providers are available for %s%s (%s)", event._is_runtime and "runtime " or "", + event._item, + ", ".join(event._candidates)) + logger.info("consider defining a PREFERRED_PROVIDER entry to match %s", event._item) + continue + + if isinstance(event, bb.event.NoProvider): + errors = errors + 1 + if event._runtime: + r = "R" + else: + r = "" + + if event._dependees: + text = "Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)" % (r, event._item, ", ".join(event._dependees), r) + else: + text = "Nothing %sPROVIDES '%s'" % (r, event._item) + + logger.error(text) + if event._reasons: + for reason in event._reasons: + logger.error("%s", reason) + text += reason + buildinfohelper.store_log_error(text) + continue + + if isinstance(event, bb.event.ConfigParsed): + continue + if isinstance(event, bb.event.RecipeParsed): + continue + + # end of saved events + + if isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)): + buildinfohelper.store_started_task(event) + continue + + if isinstance(event, bb.runqueue.runQueueTaskCompleted): + buildinfohelper.update_and_store_task(event) + continue + + if isinstance(event, bb.runqueue.runQueueTaskFailed): + buildinfohelper.update_and_store_task(event) + taskfailures.append(event.taskstring) + logger.error("Task %s (%s) failed with exit code '%s'", + event.taskid, event.taskstring, event.exitcode) + continue + + if isinstance(event, (bb.runqueue.sceneQueueTaskCompleted, bb.runqueue.sceneQueueTaskFailed)): + buildinfohelper.update_and_store_task(event) + continue + + + if isinstance(event, (bb.event.TreeDataPreparationStarted, bb.event.TreeDataPreparationCompleted)): + continue + + if isinstance(event, (bb.event.BuildCompleted, bb.command.CommandFailed)): + + errorcode = 0 + if isinstance(event, bb.command.CommandFailed): + errors += 1 + errorcode = 1 + logger.error("Command execution failed: %s", event.error) + + # turn off logging to the current build log + _close_build_log(build_log) + + # reset ready for next BuildStarted + build_log = None + + # update the build info helper on BuildCompleted, not on CommandXXX + buildinfohelper.update_build_information(event, errors, warnings, taskfailures) + + brbe = buildinfohelper.brbe + buildinfohelper.close(errorcode) + + # we start a new build info + if params.observe_only: + logger.debug("ToasterUI prepared for new build") + errors = 0 + warnings = 0 + taskfailures = [] + buildinfohelper = BuildInfoHelper(server, build_history_enabled) + else: + main.shutdown = 1 + + logger.info("ToasterUI build done, brbe: %s", brbe) + continue + + if isinstance(event, (bb.command.CommandCompleted, + bb.command.CommandFailed, + bb.command.CommandExit)): + if params.observe_only: + errorcode = 0 + else: + main.shutdown = 1 + + continue + + if isinstance(event, bb.event.MetadataEvent): + if event.type == "SinglePackageInfo": + buildinfohelper.store_build_package_information(event) + elif event.type == "LayerInfo": + buildinfohelper.store_layer_info(event) + elif event.type == "BuildStatsList": + buildinfohelper.store_tasks_stats(event) + elif event.type == "ImagePkgList": + buildinfohelper.store_target_package_data(event) + elif event.type == "MissedSstate": + buildinfohelper.store_missed_state_tasks(event) + elif event.type == "ImageFileSize": + buildinfohelper.update_target_image_file(event) + elif event.type == "ArtifactFileSize": + buildinfohelper.update_artifact_image_file(event) + elif event.type == "LicenseManifestPath": + buildinfohelper.store_license_manifest_path(event) + elif event.type == "SetBRBE": + buildinfohelper.brbe = buildinfohelper._get_data_from_event(event) + elif event.type == "OSErrorException": + logger.error(event) + else: + logger.error("Unprocessed MetadataEvent %s ", str(event)) + continue + + if isinstance(event, bb.cooker.CookerExit): + # shutdown when bitbake server shuts down + main.shutdown = 1 + continue + + if isinstance(event, bb.event.DepTreeGenerated): + buildinfohelper.store_dependency_information(event) + continue + + logger.warn("Unknown event: %s", event) + return_value += 1 + + except EnvironmentError as ioerror: + # ignore interrupted io + if ioerror.args[0] == 4: + pass + except KeyboardInterrupt: + main.shutdown = 1 + except Exception as e: + # print errors to log + import traceback + from pprint import pformat + exception_data = traceback.format_exc() + logger.error("%s\n%s" , e, exception_data) + + # save them to database, if possible; if it fails, we already logged to console. + try: + buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data)) + except Exception as ce: + logger.error("CRITICAL - Failed to to save toaster exception to the database: %s", str(ce)) + + # make sure we return with an error + return_value += 1 + + if interrupted and return_value == 0: + return_value += 1 + + logger.warn("Return value is %d", return_value) + return return_value diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/uievent.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/uievent.py new file mode 100644 index 000000000..df093c53c --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/uievent.py @@ -0,0 +1,161 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer +# Copyright (C) 2006 - 2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +""" +Use this class to fork off a thread to recieve event callbacks from the bitbake +server and queue them for the UI to process. This process must be used to avoid +client/server deadlocks. +""" + +import socket, threading, pickle, collections +from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler + +class BBUIEventQueue: + def __init__(self, BBServer, clientinfo=("localhost, 0")): + + self.eventQueue = [] + self.eventQueueLock = threading.Lock() + self.eventQueueNotify = threading.Event() + + self.BBServer = BBServer + self.clientinfo = clientinfo + + server = UIXMLRPCServer(self.clientinfo) + self.host, self.port = server.socket.getsockname() + + server.register_function( self.system_quit, "event.quit" ) + server.register_function( self.send_event, "event.sendpickle" ) + server.socket.settimeout(1) + + self.EventHandle = None + + # the event handler registration may fail here due to cooker being in invalid state + # this is a transient situation, and we should retry a couple of times before + # giving up + + for count_tries in range(5): + ret = self.BBServer.registerEventHandler(self.host, self.port) + + if isinstance(ret, collections.Iterable): + self.EventHandle, error = ret + else: + self.EventHandle = ret + error = "" + + if self.EventHandle != None: + break + + errmsg = "Could not register UI event handler. Error: %s, host %s, "\ + "port %d" % (error, self.host, self.port) + bb.warn("%s, retry" % errmsg) + + import time + time.sleep(1) + else: + raise Exception(errmsg) + + self.server = server + + self.t = threading.Thread() + self.t.setDaemon(True) + self.t.run = self.startCallbackHandler + self.t.start() + + def getEvent(self): + + self.eventQueueLock.acquire() + + if len(self.eventQueue) == 0: + self.eventQueueLock.release() + return None + + item = self.eventQueue.pop(0) + + if len(self.eventQueue) == 0: + self.eventQueueNotify.clear() + + self.eventQueueLock.release() + return item + + def waitEvent(self, delay): + self.eventQueueNotify.wait(delay) + return self.getEvent() + + def queue_event(self, event): + self.eventQueueLock.acquire() + self.eventQueue.append(event) + self.eventQueueNotify.set() + self.eventQueueLock.release() + + def send_event(self, event): + self.queue_event(pickle.loads(event)) + + def startCallbackHandler(self): + + self.server.timeout = 1 + bb.utils.set_process_name("UIEventQueue") + while not self.server.quit: + try: + self.server.handle_request() + except Exception as e: + import traceback + logger.error("BBUIEventQueue.startCallbackHandler: Exception while trying to handle request: %s\n%s" % (e, traceback.format_exc(e))) + + self.server.server_close() + + def system_quit( self ): + """ + Shut down the callback thread + """ + try: + self.BBServer.unregisterEventHandler(self.EventHandle) + except: + pass + self.server.quit = True + +class UIXMLRPCServer (SimpleXMLRPCServer): + + def __init__( self, interface ): + self.quit = False + SimpleXMLRPCServer.__init__( self, + interface, + requestHandler=SimpleXMLRPCRequestHandler, + logRequests=False, allow_none=True) + + def get_request(self): + while not self.quit: + try: + sock, addr = self.socket.accept() + sock.settimeout(1) + return (sock, addr) + except socket.timeout: + pass + return (None, None) + + def close_request(self, request): + if request is None: + return + SimpleXMLRPCServer.close_request(self, request) + + def process_request(self, request, client_address): + if request is None: + return + SimpleXMLRPCServer.process_request(self, request, client_address) + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/uihelper.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/uihelper.py new file mode 100644 index 000000000..db70b763f --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/uihelper.py @@ -0,0 +1,59 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer +# Copyright (C) 2006 - 2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import bb.build + +class BBUIHelper: + def __init__(self): + self.needUpdate = False + self.running_tasks = {} + # Running PIDs preserves the order tasks were executed in + self.running_pids = [] + self.failed_tasks = [] + self.tasknumber_current = 0 + self.tasknumber_total = 0 + + def eventHandler(self, event): + if isinstance(event, bb.build.TaskStarted): + self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task) } + self.running_pids.append(event.pid) + self.needUpdate = True + if isinstance(event, bb.build.TaskSucceeded): + del self.running_tasks[event.pid] + self.running_pids.remove(event.pid) + self.needUpdate = True + if isinstance(event, bb.build.TaskFailedSilent): + del self.running_tasks[event.pid] + self.running_pids.remove(event.pid) + # Don't add to the failed tasks list since this is e.g. a setscene task failure + self.needUpdate = True + if isinstance(event, bb.build.TaskFailed): + del self.running_tasks[event.pid] + self.running_pids.remove(event.pid) + self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)}) + self.needUpdate = True + if isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted): + self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1 + self.tasknumber_total = event.stats.total + self.needUpdate = True + + def getTasks(self): + self.needUpdate = False + return (self.running_tasks, self.failed_tasks) + diff --git a/import-layers/yocto-poky/bitbake/lib/bb/utils.py b/import-layers/yocto-poky/bitbake/lib/bb/utils.py new file mode 100644 index 000000000..3544bbe17 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/utils.py @@ -0,0 +1,1453 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake Utility Functions +""" + +# Copyright (C) 2004 Michael Lauer +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import re, fcntl, os, string, stat, shutil, time +import sys +import errno +import logging +import bb +import bb.msg +import multiprocessing +import fcntl +import subprocess +import glob +import fnmatch +import traceback +import errno +import signal +import ast +from commands import getstatusoutput +from contextlib import contextmanager +from ctypes import cdll + + +logger = logging.getLogger("BitBake.Util") + +def clean_context(): + return { + "os": os, + "bb": bb, + "time": time, + } + +def get_context(): + return _context + + +def set_context(ctx): + _context = ctx + +# Context used in better_exec, eval +_context = clean_context() + +class VersionStringException(Exception): + """Exception raised when an invalid version specification is found""" + +def explode_version(s): + r = [] + alpha_regexp = re.compile('^([a-zA-Z]+)(.*)$') + numeric_regexp = re.compile('^(\d+)(.*)$') + while (s != ''): + if s[0] in string.digits: + m = numeric_regexp.match(s) + r.append((0, int(m.group(1)))) + s = m.group(2) + continue + if s[0] in string.letters: + m = alpha_regexp.match(s) + r.append((1, m.group(1))) + s = m.group(2) + continue + if s[0] == '~': + r.append((-1, s[0])) + else: + r.append((2, s[0])) + s = s[1:] + return r + +def split_version(s): + """Split a version string into its constituent parts (PE, PV, PR)""" + s = s.strip(" <>=") + e = 0 + if s.count(':'): + e = int(s.split(":")[0]) + s = s.split(":")[1] + r = "" + if s.count('-'): + r = s.rsplit("-", 1)[1] + s = s.rsplit("-", 1)[0] + v = s + return (e, v, r) + +def vercmp_part(a, b): + va = explode_version(a) + vb = explode_version(b) + while True: + if va == []: + (oa, ca) = (0, None) + else: + (oa, ca) = va.pop(0) + if vb == []: + (ob, cb) = (0, None) + else: + (ob, cb) = vb.pop(0) + if (oa, ca) == (0, None) and (ob, cb) == (0, None): + return 0 + if oa < ob: + return -1 + elif oa > ob: + return 1 + elif ca < cb: + return -1 + elif ca > cb: + return 1 + +def vercmp(ta, tb): + (ea, va, ra) = ta + (eb, vb, rb) = tb + + r = int(ea or 0) - int(eb or 0) + if (r == 0): + r = vercmp_part(va, vb) + if (r == 0): + r = vercmp_part(ra, rb) + return r + +def vercmp_string(a, b): + ta = split_version(a) + tb = split_version(b) + return vercmp(ta, tb) + +def vercmp_string_op(a, b, op): + """ + Compare two versions and check if the specified comparison operator matches the result of the comparison. + This function is fairly liberal about what operators it will accept since there are a variety of styles + depending on the context. + """ + res = vercmp_string(a, b) + if op in ('=', '=='): + return res == 0 + elif op == '<=': + return res <= 0 + elif op == '>=': + return res >= 0 + elif op in ('>', '>>'): + return res > 0 + elif op in ('<', '<<'): + return res < 0 + elif op == '!=': + return res != 0 + else: + raise VersionStringException('Unsupported comparison operator "%s"' % op) + +def explode_deps(s): + """ + Take an RDEPENDS style string of format: + "DEPEND1 (optional version) DEPEND2 (optional version) ..." + and return a list of dependencies. + Version information is ignored. + """ + r = [] + l = s.split() + flag = False + for i in l: + if i[0] == '(': + flag = True + #j = [] + if not flag: + r.append(i) + #else: + # j.append(i) + if flag and i.endswith(')'): + flag = False + # Ignore version + #r[-1] += ' ' + ' '.join(j) + return r + +def explode_dep_versions2(s): + """ + Take an RDEPENDS style string of format: + "DEPEND1 (optional version) DEPEND2 (optional version) ..." + and return a dictionary of dependencies and versions. + """ + r = {} + l = s.replace(",", "").split() + lastdep = None + lastcmp = "" + lastver = "" + incmp = False + inversion = False + for i in l: + if i[0] == '(': + incmp = True + i = i[1:].strip() + if not i: + continue + + if incmp: + incmp = False + inversion = True + # This list is based on behavior and supported comparisons from deb, opkg and rpm. + # + # Even though =<, <<, ==, !=, =>, and >> may not be supported, + # we list each possibly valid item. + # The build system is responsible for validation of what it supports. + if i.startswith(('<=', '=<', '<<', '==', '!=', '>=', '=>', '>>')): + lastcmp = i[0:2] + i = i[2:] + elif i.startswith(('<', '>', '=')): + lastcmp = i[0:1] + i = i[1:] + else: + # This is an unsupported case! + raise VersionStringException('Invalid version specification in "(%s" - invalid or missing operator' % i) + lastcmp = (i or "") + i = "" + i.strip() + if not i: + continue + + if inversion: + if i.endswith(')'): + i = i[:-1] or "" + inversion = False + if lastver and i: + lastver += " " + if i: + lastver += i + if lastdep not in r: + r[lastdep] = [] + r[lastdep].append(lastcmp + " " + lastver) + continue + + #if not inversion: + lastdep = i + lastver = "" + lastcmp = "" + if not (i in r and r[i]): + r[lastdep] = [] + + return r + +def explode_dep_versions(s): + r = explode_dep_versions2(s) + for d in r: + if not r[d]: + r[d] = None + continue + if len(r[d]) > 1: + bb.warn("explode_dep_versions(): Item %s appeared in dependency string '%s' multiple times with different values. explode_dep_versions cannot cope with this." % (d, s)) + r[d] = r[d][0] + return r + +def join_deps(deps, commasep=True): + """ + Take the result from explode_dep_versions and generate a dependency string + """ + result = [] + for dep in deps: + if deps[dep]: + if isinstance(deps[dep], list): + for v in deps[dep]: + result.append(dep + " (" + v + ")") + else: + result.append(dep + " (" + deps[dep] + ")") + else: + result.append(dep) + if commasep: + return ", ".join(result) + else: + return " ".join(result) + +def _print_trace(body, line): + """ + Print the Environment of a Text Body + """ + error = [] + # print the environment of the method + min_line = max(1, line-4) + max_line = min(line + 4, len(body)) + for i in range(min_line, max_line + 1): + if line == i: + error.append(' *** %.4d:%s' % (i, body[i-1].rstrip())) + else: + error.append(' %.4d:%s' % (i, body[i-1].rstrip())) + return error + +def better_compile(text, file, realfile, mode = "exec", lineno = 0): + """ + A better compile method. This method + will print the offending lines. + """ + try: + cache = bb.methodpool.compile_cache(text) + if cache: + return cache + # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though + text2 = "\n" * int(lineno) + text + code = compile(text2, realfile, mode) + bb.methodpool.compile_cache_add(text, code) + return code + except Exception as e: + error = [] + # split the text into lines again + body = text.split('\n') + error.append("Error in compiling python function in %s, line %s:\n" % (realfile, lineno)) + if hasattr(e, "lineno"): + error.append("The code lines resulting in this error were:") + error.extend(_print_trace(body, e.lineno)) + else: + error.append("The function causing this error was:") + for line in body: + error.append(line) + error.append("%s: %s" % (e.__class__.__name__, str(e))) + + logger.error("\n".join(error)) + + e = bb.BBHandledException(e) + raise e + +def _print_exception(t, value, tb, realfile, text, context): + error = [] + try: + exception = traceback.format_exception_only(t, value) + error.append('Error executing a python function in %s:\n' % realfile) + + # Strip 'us' from the stack (better_exec call) unless that was where the + # error came from + if tb.tb_next is not None: + tb = tb.tb_next + + textarray = text.split('\n') + + linefailed = tb.tb_lineno + + tbextract = traceback.extract_tb(tb) + tbformat = traceback.format_list(tbextract) + error.append("The stack trace of python calls that resulted in this exception/failure was:") + error.append("File: '%s', lineno: %s, function: %s" % (tbextract[0][0], tbextract[0][1], tbextract[0][2])) + error.extend(_print_trace(textarray, linefailed)) + + # See if this is a function we constructed and has calls back into other functions in + # "text". If so, try and improve the context of the error by diving down the trace + level = 0 + nexttb = tb.tb_next + while nexttb is not None and (level+1) < len(tbextract): + error.append("File: '%s', lineno: %s, function: %s" % (tbextract[level+1][0], tbextract[level+1][1], tbextract[level+1][2])) + if tbextract[level][0] == tbextract[level+1][0] and tbextract[level+1][2] == tbextract[level][0]: + # The code was possibly in the string we compiled ourselves + error.extend(_print_trace(textarray, tbextract[level+1][1])) + elif tbextract[level+1][0].startswith("/"): + # The code looks like it might be in a file, try and load it + try: + with open(tbextract[level+1][0], "r") as f: + text = f.readlines() + error.extend(_print_trace(text, tbextract[level+1][1])) + except: + error.append(tbformat[level+1]) + else: + error.append(tbformat[level+1]) + nexttb = tb.tb_next + level = level + 1 + + error.append("Exception: %s" % ''.join(exception)) + finally: + logger.error("\n".join(error)) + +def better_exec(code, context, text = None, realfile = "<code>", pythonexception=False): + """ + Similiar to better_compile, better_exec will + print the lines that are responsible for the + error. + """ + import bb.parse + if not text: + text = code + if not hasattr(code, "co_filename"): + code = better_compile(code, realfile, realfile) + try: + exec(code, get_context(), context) + except (bb.BBHandledException, bb.parse.SkipRecipe, bb.build.FuncFailed, bb.data_smart.ExpansionError): + # Error already shown so passthrough, no need for traceback + raise + except Exception as e: + if pythonexception: + raise + (t, value, tb) = sys.exc_info() + try: + _print_exception(t, value, tb, realfile, text, context) + except Exception as e: + logger.error("Exception handler error: %s" % str(e)) + + e = bb.BBHandledException(e) + raise e + +def simple_exec(code, context): + exec(code, get_context(), context) + +def better_eval(source, locals): + return eval(source, get_context(), locals) + +@contextmanager +def fileslocked(files): + """Context manager for locking and unlocking file locks.""" + locks = [] + if files: + for lockfile in files: + locks.append(bb.utils.lockfile(lockfile)) + + yield + + for lock in locks: + bb.utils.unlockfile(lock) + +@contextmanager +def timeout(seconds): + def timeout_handler(signum, frame): + pass + + original_handler = signal.signal(signal.SIGALRM, timeout_handler) + + try: + signal.alarm(seconds) + yield + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, original_handler) + +def lockfile(name, shared=False, retry=True, block=False): + """ + Use the specified file as a lock file, return when the lock has + been acquired. Returns a variable to pass to unlockfile(). + Parameters: + retry: True to re-try locking if it fails, False otherwise + block: True to block until the lock succeeds, False otherwise + The retry and block parameters are kind of equivalent unless you + consider the possibility of sending a signal to the process to break + out - at which point you want block=True rather than retry=True. + """ + dirname = os.path.dirname(name) + mkdirhier(dirname) + + if not os.access(dirname, os.W_OK): + logger.error("Unable to acquire lock '%s', directory is not writable", + name) + sys.exit(1) + + op = fcntl.LOCK_EX + if shared: + op = fcntl.LOCK_SH + if not retry and not block: + op = op | fcntl.LOCK_NB + + while True: + # If we leave the lockfiles lying around there is no problem + # but we should clean up after ourselves. This gives potential + # for races though. To work around this, when we acquire the lock + # we check the file we locked was still the lock file on disk. + # by comparing inode numbers. If they don't match or the lockfile + # no longer exists, we start again. + + # This implementation is unfair since the last person to request the + # lock is the most likely to win it. + + try: + lf = open(name, 'a+') + fileno = lf.fileno() + fcntl.flock(fileno, op) + statinfo = os.fstat(fileno) + if os.path.exists(lf.name): + statinfo2 = os.stat(lf.name) + if statinfo.st_ino == statinfo2.st_ino: + return lf + lf.close() + except Exception: + try: + lf.close() + except Exception: + pass + pass + if not retry: + return None + +def unlockfile(lf): + """ + Unlock a file locked using lockfile() + """ + try: + # If we had a shared lock, we need to promote to exclusive before + # removing the lockfile. Attempt this, ignore failures. + fcntl.flock(lf.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB) + os.unlink(lf.name) + except (IOError, OSError): + pass + fcntl.flock(lf.fileno(), fcntl.LOCK_UN) + lf.close() + +def md5_file(filename): + """ + Return the hex string representation of the MD5 checksum of filename. + """ + try: + import hashlib + m = hashlib.md5() + except ImportError: + import md5 + m = md5.new() + + with open(filename, "rb") as f: + for line in f: + m.update(line) + return m.hexdigest() + +def sha256_file(filename): + """ + Return the hex string representation of the 256-bit SHA checksum of + filename. On Python 2.4 this will return None, so callers will need to + handle that by either skipping SHA checks, or running a standalone sha256sum + binary. + """ + try: + import hashlib + except ImportError: + return None + + s = hashlib.sha256() + with open(filename, "rb") as f: + for line in f: + s.update(line) + return s.hexdigest() + +def sha1_file(filename): + """ + Return the hex string representation of the SHA1 checksum of the filename + """ + try: + import hashlib + except ImportError: + return None + + s = hashlib.sha1() + with open(filename, "rb") as f: + for line in f: + s.update(line) + return s.hexdigest() + +def preserved_envvars_exported(): + """Variables which are taken from the environment and placed in and exported + from the metadata""" + return [ + 'BB_TASKHASH', + 'HOME', + 'LOGNAME', + 'PATH', + 'PWD', + 'SHELL', + 'TERM', + 'USER', + ] + +def preserved_envvars(): + """Variables which are taken from the environment and placed in the metadata""" + v = [ + 'BBPATH', + 'BB_PRESERVE_ENV', + 'BB_ENV_WHITELIST', + 'BB_ENV_EXTRAWHITE', + ] + return v + preserved_envvars_exported() + +def filter_environment(good_vars): + """ + Create a pristine environment for bitbake. This will remove variables that + are not known and may influence the build in a negative way. + """ + + removed_vars = {} + for key in os.environ.keys(): + if key in good_vars: + continue + + removed_vars[key] = os.environ[key] + os.unsetenv(key) + del os.environ[key] + + if removed_vars: + logger.debug(1, "Removed the following variables from the environment: %s", ", ".join(removed_vars.keys())) + + return removed_vars + +def approved_variables(): + """ + Determine and return the list of whitelisted variables which are approved + to remain in the environment. + """ + if 'BB_PRESERVE_ENV' in os.environ: + return os.environ.keys() + approved = [] + if 'BB_ENV_WHITELIST' in os.environ: + approved = os.environ['BB_ENV_WHITELIST'].split() + approved.extend(['BB_ENV_WHITELIST']) + else: + approved = preserved_envvars() + if 'BB_ENV_EXTRAWHITE' in os.environ: + approved.extend(os.environ['BB_ENV_EXTRAWHITE'].split()) + if 'BB_ENV_EXTRAWHITE' not in approved: + approved.extend(['BB_ENV_EXTRAWHITE']) + return approved + +def clean_environment(): + """ + Clean up any spurious environment variables. This will remove any + variables the user hasn't chosen to preserve. + """ + if 'BB_PRESERVE_ENV' not in os.environ: + good_vars = approved_variables() + return filter_environment(good_vars) + + return {} + +def empty_environment(): + """ + Remove all variables from the environment. + """ + for s in os.environ.keys(): + os.unsetenv(s) + del os.environ[s] + +def build_environment(d): + """ + Build an environment from all exported variables. + """ + import bb.data + for var in bb.data.keys(d): + export = d.getVarFlag(var, "export", False) + if export: + os.environ[var] = d.getVar(var, True) or "" + +def _check_unsafe_delete_path(path): + """ + Basic safeguard against recursively deleting something we shouldn't. If it returns True, + the caller should raise an exception with an appropriate message. + NOTE: This is NOT meant to be a security mechanism - just a guard against silly mistakes + with potentially disastrous results. + """ + extra = '' + # HOME might not be /home/something, so in case we can get it, check against it + homedir = os.environ.get('HOME', '') + if homedir: + extra = '|%s' % homedir + if re.match('(/|//|/home|/home/[^/]*%s)$' % extra, os.path.abspath(path)): + return True + return False + +def remove(path, recurse=False): + """Equivalent to rm -f or rm -rf""" + if not path: + return + if recurse: + for name in glob.glob(path): + if _check_unsafe_delete_path(path): + raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % path) + # shutil.rmtree(name) would be ideal but its too slow + subprocess.call(['rm', '-rf'] + glob.glob(path)) + return + for name in glob.glob(path): + try: + os.unlink(name) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + +def prunedir(topdir): + # Delete everything reachable from the directory named in 'topdir'. + # CAUTION: This is dangerous! + if _check_unsafe_delete_path(topdir): + raise Exception('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!' % topdir) + for root, dirs, files in os.walk(topdir, topdown = False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + if os.path.islink(os.path.join(root, name)): + os.remove(os.path.join(root, name)) + else: + os.rmdir(os.path.join(root, name)) + os.rmdir(topdir) + +# +# Could also use return re.compile("(%s)" % "|".join(map(re.escape, suffixes))).sub(lambda mo: "", var) +# but thats possibly insane and suffixes is probably going to be small +# +def prune_suffix(var, suffixes, d): + # See if var ends with any of the suffixes listed and + # remove it if found + for suffix in suffixes: + if var.endswith(suffix): + return var.replace(suffix, "") + return var + +def mkdirhier(directory): + """Create a directory like 'mkdir -p', but does not complain if + directory already exists like os.makedirs + """ + + try: + os.makedirs(directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + +def movefile(src, dest, newmtime = None, sstat = None): + """Moves a file from src to dest, preserving all permissions and + attributes; mtime will be preserved even when moving across + filesystems. Returns true on success and false on failure. Move is + atomic. + """ + + #print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")" + try: + if not sstat: + sstat = os.lstat(src) + except Exception as e: + print("movefile: Stating source file failed...", e) + return None + + destexists = 1 + try: + dstat = os.lstat(dest) + except: + dstat = os.lstat(os.path.dirname(dest)) + destexists = 0 + + if destexists: + if stat.S_ISLNK(dstat[stat.ST_MODE]): + try: + os.unlink(dest) + destexists = 0 + except Exception as e: + pass + + if stat.S_ISLNK(sstat[stat.ST_MODE]): + try: + target = os.readlink(src) + if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): + os.unlink(dest) + os.symlink(target, dest) + #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) + os.unlink(src) + return os.lstat(dest) + except Exception as e: + print("movefile: failed to properly create symlink:", dest, "->", target, e) + return None + + renamefailed = 1 + if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]: + try: + # os.rename needs to know the dest path ending with file name + # so append the file name to a path only if it's a dir specified + srcfname = os.path.basename(src) + destpath = os.path.join(dest, srcfname) if os.path.isdir(dest) \ + else dest + os.rename(src, destpath) + renamefailed = 0 + except Exception as e: + if e[0] != errno.EXDEV: + # Some random error. + print("movefile: Failed to move", src, "to", dest, e) + return None + # Invalid cross-device-link 'bind' mounted or actually Cross-Device + + if renamefailed: + didcopy = 0 + if stat.S_ISREG(sstat[stat.ST_MODE]): + try: # For safety copy then move it over. + shutil.copyfile(src, dest + "#new") + os.rename(dest + "#new", dest) + didcopy = 1 + except Exception as e: + print('movefile: copy', src, '->', dest, 'failed.', e) + return None + else: + #we don't yet handle special, so we need to fall back to /bin/mv + a = getstatusoutput("/bin/mv -f " + "'" + src + "' '" + dest + "'") + if a[0] != 0: + print("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a) + return None # failure + try: + if didcopy: + os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID]) + os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown + os.unlink(src) + except Exception as e: + print("movefile: Failed to chown/chmod/unlink", dest, e) + return None + + if newmtime: + os.utime(dest, (newmtime, newmtime)) + else: + os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME])) + newmtime = sstat[stat.ST_MTIME] + return newmtime + +def copyfile(src, dest, newmtime = None, sstat = None): + """ + Copies a file from src to dest, preserving all permissions and + attributes; mtime will be preserved even when moving across + filesystems. Returns true on success and false on failure. + """ + #print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")" + try: + if not sstat: + sstat = os.lstat(src) + except Exception as e: + logger.warn("copyfile: stat of %s failed (%s)" % (src, e)) + return False + + destexists = 1 + try: + dstat = os.lstat(dest) + except: + dstat = os.lstat(os.path.dirname(dest)) + destexists = 0 + + if destexists: + if stat.S_ISLNK(dstat[stat.ST_MODE]): + try: + os.unlink(dest) + destexists = 0 + except Exception as e: + pass + + if stat.S_ISLNK(sstat[stat.ST_MODE]): + try: + target = os.readlink(src) + if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): + os.unlink(dest) + os.symlink(target, dest) + #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) + return os.lstat(dest) + except Exception as e: + logger.warn("copyfile: failed to create symlink %s to %s (%s)" % (dest, target, e)) + return False + + if stat.S_ISREG(sstat[stat.ST_MODE]): + try: + srcchown = False + if not os.access(src, os.R_OK): + # Make sure we can read it + srcchown = True + os.chmod(src, sstat[stat.ST_MODE] | stat.S_IRUSR) + + # For safety copy then move it over. + shutil.copyfile(src, dest + "#new") + os.rename(dest + "#new", dest) + except Exception as e: + logger.warn("copyfile: copy %s to %s failed (%s)" % (src, dest, e)) + return False + finally: + if srcchown: + os.chmod(src, sstat[stat.ST_MODE]) + os.utime(src, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME])) + + else: + #we don't yet handle special, so we need to fall back to /bin/mv + a = getstatusoutput("/bin/cp -f " + "'" + src + "' '" + dest + "'") + if a[0] != 0: + logger.warn("copyfile: failed to copy special file %s to %s (%s)" % (src, dest, a)) + return False # failure + try: + os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID]) + os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown + except Exception as e: + logger.warn("copyfile: failed to chown/chmod %s (%s)" % (dest, e)) + return False + + if newmtime: + os.utime(dest, (newmtime, newmtime)) + else: + os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME])) + newmtime = sstat[stat.ST_MTIME] + return newmtime + +def which(path, item, direction = 0, history = False): + """ + Locate a file in a PATH + """ + + hist = [] + paths = (path or "").split(':') + if direction != 0: + paths.reverse() + + for p in paths: + next = os.path.join(p, item) + hist.append(next) + if os.path.exists(next): + if not os.path.isabs(next): + next = os.path.abspath(next) + if history: + return next, hist + return next + + if history: + return "", hist + return "" + +def to_boolean(string, default=None): + if not string: + return default + + normalized = string.lower() + if normalized in ("y", "yes", "1", "true"): + return True + elif normalized in ("n", "no", "0", "false"): + return False + else: + raise ValueError("Invalid value for to_boolean: %s" % string) + +def contains(variable, checkvalues, truevalue, falsevalue, d): + """Check if a variable contains all the values specified. + + Arguments: + + variable -- the variable name. This will be fetched and expanded (using + d.getVar(variable, True)) and then split into a set(). + + checkvalues -- if this is a string it is split on whitespace into a set(), + otherwise coerced directly into a set(). + + truevalue -- the value to return if checkvalues is a subset of variable. + + falsevalue -- the value to return if variable is empty or if checkvalues is + not a subset of variable. + + d -- the data store. + """ + + val = d.getVar(variable, True) + if not val: + return falsevalue + val = set(val.split()) + if isinstance(checkvalues, basestring): + checkvalues = set(checkvalues.split()) + else: + checkvalues = set(checkvalues) + if checkvalues.issubset(val): + return truevalue + return falsevalue + +def contains_any(variable, checkvalues, truevalue, falsevalue, d): + val = d.getVar(variable, True) + if not val: + return falsevalue + val = set(val.split()) + if isinstance(checkvalues, basestring): + checkvalues = set(checkvalues.split()) + else: + checkvalues = set(checkvalues) + if checkvalues & val: + return truevalue + return falsevalue + +def cpu_count(): + return multiprocessing.cpu_count() + +def nonblockingfd(fd): + fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) + +def process_profilelog(fn, pout = None): + # Either call with a list of filenames and set pout or a filename and optionally pout. + if not pout: + pout = fn + '.processed' + pout = open(pout, 'w') + + import pstats + if isinstance(fn, list): + p = pstats.Stats(*fn, stream=pout) + else: + p = pstats.Stats(fn, stream=pout) + p.sort_stats('time') + p.print_stats() + p.print_callers() + p.sort_stats('cumulative') + p.print_stats() + + pout.flush() + pout.close() + +# +# Was present to work around multiprocessing pool bugs in python < 2.7.3 +# +def multiprocessingpool(*args, **kwargs): + + import multiprocessing.pool + #import multiprocessing.util + #multiprocessing.util.log_to_stderr(10) + # Deal with a multiprocessing bug where signals to the processes would be delayed until the work + # completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed. + def wrapper(func): + def wrap(self, timeout=None): + return func(self, timeout=timeout if timeout is not None else 1e100) + return wrap + multiprocessing.pool.IMapIterator.next = wrapper(multiprocessing.pool.IMapIterator.next) + + return multiprocessing.Pool(*args, **kwargs) + +def exec_flat_python_func(func, *args, **kwargs): + """Execute a flat python function (defined with def funcname(args):...)""" + # Prepare a small piece of python code which calls the requested function + # To do this we need to prepare two things - a set of variables we can use to pass + # the values of arguments into the calling function, and the list of arguments for + # the function being called + context = {} + funcargs = [] + # Handle unnamed arguments + aidx = 1 + for arg in args: + argname = 'arg_%s' % aidx + context[argname] = arg + funcargs.append(argname) + aidx += 1 + # Handle keyword arguments + context.update(kwargs) + funcargs.extend(['%s=%s' % (arg, arg) for arg in kwargs.iterkeys()]) + code = 'retval = %s(%s)' % (func, ', '.join(funcargs)) + comp = bb.utils.better_compile(code, '<string>', '<string>') + bb.utils.better_exec(comp, context, code, '<string>') + return context['retval'] + +def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): + """Edit lines from a recipe or config file and modify one or more + specified variable values set in the file using a specified callback + function. Lines are expected to have trailing newlines. + Parameters: + meta_lines: lines from the file; can be a list or an iterable + (e.g. file pointer) + variables: a list of variable names to look for. Functions + may also be specified, but must be specified with '()' at + the end of the name. Note that the function doesn't have + any intrinsic understanding of _append, _prepend, _remove, + or overrides, so these are considered as part of the name. + These values go into a regular expression, so regular + expression syntax is allowed. + varfunc: callback function called for every variable matching + one of the entries in the variables parameter. The function + should take four arguments: + varname: name of variable matched + origvalue: current value in file + op: the operator (e.g. '+=') + newlines: list of lines up to this point. You can use + this to prepend lines before this variable setting + if you wish. + and should return a three-element tuple: + newvalue: new value to substitute in, or None to drop + the variable setting entirely. (If the removal + results in two consecutive blank lines, one of the + blank lines will also be dropped). + newop: the operator to use - if you specify None here, + the original operation will be used. + indent: number of spaces to indent multi-line entries, + or -1 to indent up to the level of the assignment + and opening quote, or a string to use as the indent. + minbreak: True to allow the first element of a + multi-line value to continue on the same line as + the assignment, False to indent before the first + element. + match_overrides: True to match items with _overrides on the end, + False otherwise + Returns a tuple: + updated: + True if changes were made, False otherwise. + newlines: + Lines after processing + """ + + var_res = {} + if match_overrides: + override_re = '(_[a-zA-Z0-9-_$(){}]+)?' + else: + override_re = '' + for var in variables: + if var.endswith('()'): + var_res[var] = re.compile('^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re)) + else: + var_res[var] = re.compile('^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re)) + + updated = False + varset_start = '' + varlines = [] + newlines = [] + in_var = None + full_value = '' + var_end = '' + + def handle_var_end(): + prerun_newlines = newlines[:] + op = varset_start[len(in_var):].strip() + (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines) + changed = (prerun_newlines != newlines) + + if newvalue is None: + # Drop the value + return True + elif newvalue != full_value or (newop not in [None, op]): + if newop not in [None, op]: + # Callback changed the operator + varset_new = "%s %s" % (in_var, newop) + else: + varset_new = varset_start + + if isinstance(indent, (int, long)): + if indent == -1: + indentspc = ' ' * (len(varset_new) + 2) + else: + indentspc = ' ' * indent + else: + indentspc = indent + if in_var.endswith('()'): + # A function definition + if isinstance(newvalue, list): + newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue))) + else: + if not newvalue.startswith('\n'): + newvalue = '\n' + newvalue + if not newvalue.endswith('\n'): + newvalue = newvalue + '\n' + newlines.append('%s {%s}\n' % (varset_new, newvalue)) + else: + # Normal variable + if isinstance(newvalue, list): + if not newvalue: + # Empty list -> empty string + newlines.append('%s ""\n' % varset_new) + elif minbreak: + # First item on first line + if len(newvalue) == 1: + newlines.append('%s "%s"\n' % (varset_new, newvalue[0])) + else: + newlines.append('%s "%s \\\n' % (varset_new, newvalue[0])) + for item in newvalue[1:]: + newlines.append('%s%s \\\n' % (indentspc, item)) + newlines.append('%s"\n' % indentspc) + else: + # No item on first line + newlines.append('%s " \\\n' % varset_new) + for item in newvalue: + newlines.append('%s%s \\\n' % (indentspc, item)) + newlines.append('%s"\n' % indentspc) + else: + newlines.append('%s "%s"\n' % (varset_new, newvalue)) + return True + else: + # Put the old lines back where they were + newlines.extend(varlines) + # If newlines was touched by the function, we'll need to return True + return changed + + checkspc = False + + for line in meta_lines: + if in_var: + value = line.rstrip() + varlines.append(line) + if in_var.endswith('()'): + full_value += '\n' + value + else: + full_value += value[:-1] + if value.endswith(var_end): + if in_var.endswith('()'): + if full_value.count('{') - full_value.count('}') >= 0: + continue + full_value = full_value[:-1] + if handle_var_end(): + updated = True + checkspc = True + in_var = None + else: + skip = False + for (varname, var_re) in var_res.iteritems(): + res = var_re.match(line) + if res: + isfunc = varname.endswith('()') + if isfunc: + splitvalue = line.split('{', 1) + var_end = '}' + else: + var_end = res.groups()[-1] + splitvalue = line.split(var_end, 1) + varset_start = splitvalue[0].rstrip() + value = splitvalue[1].rstrip() + if not isfunc and value.endswith('\\'): + value = value[:-1] + full_value = value + varlines = [line] + in_var = res.group(1) + if isfunc: + in_var += '()' + if value.endswith(var_end): + full_value = full_value[:-1] + if handle_var_end(): + updated = True + checkspc = True + in_var = None + skip = True + break + if not skip: + if checkspc: + checkspc = False + if newlines and newlines[-1] == '\n' and line == '\n': + # Squash blank line if there are two consecutive blanks after a removal + continue + newlines.append(line) + return (updated, newlines) + + +def edit_metadata_file(meta_file, variables, varfunc): + """Edit a recipe or config file and modify one or more specified + variable values set in the file using a specified callback function. + The file is only written to if the value(s) actually change. + This is basically the file version of edit_metadata(), see that + function's description for parameter/usage information. + Returns True if the file was written to, False otherwise. + """ + with open(meta_file, 'r') as f: + (updated, newlines) = edit_metadata(f, variables, varfunc) + if updated: + with open(meta_file, 'w') as f: + f.writelines(newlines) + return updated + + +def edit_bblayers_conf(bblayers_conf, add, remove): + """Edit bblayers.conf, adding and/or removing layers + Parameters: + bblayers_conf: path to bblayers.conf file to edit + add: layer path (or list of layer paths) to add; None or empty + list to add nothing + remove: layer path (or list of layer paths) to remove; None or + empty list to remove nothing + Returns a tuple: + notadded: list of layers specified to be added but weren't + (because they were already in the list) + notremoved: list of layers that were specified to be removed + but weren't (because they weren't in the list) + """ + + import fnmatch + + def remove_trailing_sep(pth): + if pth and pth[-1] == os.sep: + pth = pth[:-1] + return pth + + approved = bb.utils.approved_variables() + def canonicalise_path(pth): + pth = remove_trailing_sep(pth) + if 'HOME' in approved and '~' in pth: + pth = os.path.expanduser(pth) + return pth + + def layerlist_param(value): + if not value: + return [] + elif isinstance(value, list): + return [remove_trailing_sep(x) for x in value] + else: + return [remove_trailing_sep(value)] + + addlayers = layerlist_param(add) + removelayers = layerlist_param(remove) + + # Need to use a list here because we can't set non-local variables from a callback in python 2.x + bblayercalls = [] + removed = [] + plusequals = False + orig_bblayers = [] + + def handle_bblayers_firstpass(varname, origvalue, op, newlines): + bblayercalls.append(op) + if op == '=': + del orig_bblayers[:] + orig_bblayers.extend([canonicalise_path(x) for x in origvalue.split()]) + return (origvalue, None, 2, False) + + def handle_bblayers(varname, origvalue, op, newlines): + updated = False + bblayers = [remove_trailing_sep(x) for x in origvalue.split()] + if removelayers: + for removelayer in removelayers: + for layer in bblayers: + if fnmatch.fnmatch(canonicalise_path(layer), canonicalise_path(removelayer)): + updated = True + bblayers.remove(layer) + removed.append(removelayer) + break + if addlayers and not plusequals: + for addlayer in addlayers: + if addlayer not in bblayers: + updated = True + bblayers.append(addlayer) + del addlayers[:] + + if updated: + if op == '+=' and not bblayers: + bblayers = None + return (bblayers, None, 2, False) + else: + return (origvalue, None, 2, False) + + with open(bblayers_conf, 'r') as f: + (_, newlines) = edit_metadata(f, ['BBLAYERS'], handle_bblayers_firstpass) + + if not bblayercalls: + raise Exception('Unable to find BBLAYERS in %s' % bblayers_conf) + + # Try to do the "smart" thing depending on how the user has laid out + # their bblayers.conf file + if bblayercalls.count('+=') > 1: + plusequals = True + + removelayers_canon = [canonicalise_path(layer) for layer in removelayers] + notadded = [] + for layer in addlayers: + layer_canon = canonicalise_path(layer) + if layer_canon in orig_bblayers and not layer_canon in removelayers_canon: + notadded.append(layer) + notadded_canon = [canonicalise_path(layer) for layer in notadded] + addlayers[:] = [layer for layer in addlayers if canonicalise_path(layer) not in notadded_canon] + + (updated, newlines) = edit_metadata(newlines, ['BBLAYERS'], handle_bblayers) + if addlayers: + # Still need to add these + for addlayer in addlayers: + newlines.append('BBLAYERS += "%s"\n' % addlayer) + updated = True + + if updated: + with open(bblayers_conf, 'w') as f: + f.writelines(newlines) + + notremoved = list(set(removelayers) - set(removed)) + + return (notadded, notremoved) + + +def get_file_layer(filename, d): + """Determine the collection (as defined by a layer's layer.conf file) containing the specified file""" + collections = (d.getVar('BBFILE_COLLECTIONS', True) or '').split() + collection_res = {} + for collection in collections: + collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection, True) or '' + + def path_to_layer(path): + # Use longest path so we handle nested layers + matchlen = 0 + match = None + for collection, regex in collection_res.iteritems(): + if len(regex) > matchlen and re.match(regex, path): + matchlen = len(regex) + match = collection + return match + + result = None + bbfiles = (d.getVar('BBFILES', True) or '').split() + bbfilesmatch = False + for bbfilesentry in bbfiles: + if fnmatch.fnmatch(filename, bbfilesentry): + bbfilesmatch = True + result = path_to_layer(bbfilesentry) + + if not bbfilesmatch: + # Probably a bbclass + result = path_to_layer(filename) + + return result + + +# Constant taken from http://linux.die.net/include/linux/prctl.h +PR_SET_PDEATHSIG = 1 + +class PrCtlError(Exception): + pass + +def signal_on_parent_exit(signame): + """ + Trigger signame to be sent when the parent process dies + """ + signum = getattr(signal, signame) + # http://linux.die.net/man/2/prctl + result = cdll['libc.so.6'].prctl(PR_SET_PDEATHSIG, signum) + if result != 0: + raise PrCtlError('prctl failed with error code %s' % result) + +# +# Manually call the ioprio syscall. We could depend on other libs like psutil +# however this gets us enough of what we need to bitbake for now without the +# dependency +# +_unamearch = os.uname()[4] +IOPRIO_WHO_PROCESS = 1 +IOPRIO_CLASS_SHIFT = 13 + +def ioprio_set(who, cls, value): + NR_ioprio_set = None + if _unamearch == "x86_64": + NR_ioprio_set = 251 + elif _unamearch[0] == "i" and _unamearch[2:3] == "86": + NR_ioprio_set = 289 + + if NR_ioprio_set: + ioprio = value | (cls << IOPRIO_CLASS_SHIFT) + rc = cdll['libc.so.6'].syscall(NR_ioprio_set, IOPRIO_WHO_PROCESS, who, ioprio) + if rc != 0: + raise ValueError("Unable to set ioprio, syscall returned %s" % rc) + else: + bb.warn("Unable to set IO Prio for arch %s" % _unamearch) + +def set_process_name(name): + from ctypes import cdll, byref, create_string_buffer + # This is nice to have for debugging, not essential + try: + libc = cdll.LoadLibrary('libc.so.6') + buff = create_string_buffer(len(name)+1) + buff.value = name + libc.prctl(15, byref(buff), 0, 0, 0) + except: + pass + +# export common proxies variables from datastore to environment +def export_proxies(d): + import os + + variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY', + 'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY'] + exported = False + + for v in variables: + if v in os.environ.keys(): + exported = True + else: + v_proxy = d.getVar(v, True) + if v_proxy is not None: + os.environ[v] = v_proxy + exported = True + + return exported diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/AUTHORS.txt b/import-layers/yocto-poky/bitbake/lib/bs4/AUTHORS.txt new file mode 100644 index 000000000..2ac8fcc8c --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/AUTHORS.txt @@ -0,0 +1,43 @@ +Behold, mortal, the origins of Beautiful Soup... +================================================ + +Leonard Richardson is the primary programmer. + +Aaron DeVore is awesome. + +Mark Pilgrim provided the encoding detection code that forms the base +of UnicodeDammit. + +Thomas Kluyver and Ezio Melotti finished the work of getting Beautiful +Soup 4 working under Python 3. + +Simon Willison wrote soupselect, which was used to make Beautiful Soup +support CSS selectors. + +Sam Ruby helped with a lot of edge cases. + +Jonathan Ellis was awarded the prestigous Beau Potage D'Or for his +work in solving the nestable tags conundrum. + +An incomplete list of people have contributed patches to Beautiful +Soup: + + Istvan Albert, Andrew Lin, Anthony Baxter, Andrew Boyko, Tony Chang, + Zephyr Fang, Fuzzy, Roman Gaufman, Yoni Gilad, Richie Hindle, Peteris + Krumins, Kent Johnson, Ben Last, Robert Leftwich, Staffan Malmgren, + Ksenia Marasanova, JP Moins, Adam Monsen, John Nagle, "Jon", Ed + Oskiewicz, Greg Phillips, Giles Radford, Arthur Rudolph, Marko + Samastur, Jouni Seppänen, Alexander Schmolck, Andy Theyers, Glyn + Webster, Paul Wright, Danny Yoo + +An incomplete list of people who made suggestions or found bugs or +found ways to break Beautiful Soup: + + Hanno Böck, Matteo Bertini, Chris Curvey, Simon Cusack, Bruce Eckel, + Matt Ernst, Michael Foord, Tom Harris, Bill de hOra, Donald Howes, + Matt Patterson, Scott Roberts, Steve Strassmann, Mike Williams, + warchild at redho dot com, Sami Kuisma, Carlos Rocha, Bob Hutchison, + Joren Mc, Michal Migurski, John Kleven, Tim Heaney, Tripp Lilley, Ed + Summers, Dennis Sutch, Chris Smith, Aaron Sweep^W Swartz, Stuart + Turner, Greg Edwards, Kevin J Kalupson, Nikos Kouremenos, Artur de + Sousa Rocha, Yichun Wei, Per Vognsen diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/COPYING.txt b/import-layers/yocto-poky/bitbake/lib/bs4/COPYING.txt new file mode 100644 index 000000000..d668d13f0 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/COPYING.txt @@ -0,0 +1,26 @@ +Beautiful Soup is made available under the MIT license: + + Copyright (c) 2004-2012 Leonard Richardson + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE, DAMMIT. + +Beautiful Soup incorporates code from the html5lib library, which is +also made available under the MIT license. diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/NEWS.txt b/import-layers/yocto-poky/bitbake/lib/bs4/NEWS.txt new file mode 100644 index 000000000..88a60a245 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/NEWS.txt @@ -0,0 +1,1066 @@ += 4.3.2 (20131002) = + +* Fixed a bug in which short Unicode input was improperly encoded to + ASCII when checking whether or not it was the name of a file on + disk. [bug=1227016] + +* Fixed a crash when a short input contains data not valid in + filenames. [bug=1232604] + +* Fixed a bug that caused Unicode data put into UnicodeDammit to + return None instead of the original data. [bug=1214983] + +* Combined two tests to stop a spurious test failure when tests are + run by nosetests. [bug=1212445] + += 4.3.1 (20130815) = + +* Fixed yet another problem with the html5lib tree builder, caused by + html5lib's tendency to rearrange the tree during + parsing. [bug=1189267] + +* Fixed a bug that caused the optimized version of find_all() to + return nothing. [bug=1212655] + += 4.3.0 (20130812) = + +* Instead of converting incoming data to Unicode and feeding it to the + lxml tree builder in chunks, Beautiful Soup now makes successive + guesses at the encoding of the incoming data, and tells lxml to + parse the data as that encoding. Giving lxml more control over the + parsing process improves performance and avoids a number of bugs and + issues with the lxml parser which had previously required elaborate + workarounds: + + - An issue in which lxml refuses to parse Unicode strings on some + systems. [bug=1180527] + + - A returning bug that truncated documents longer than a (very + small) size. [bug=963880] + + - A returning bug in which extra spaces were added to a document if + the document defined a charset other than UTF-8. [bug=972466] + + This required a major overhaul of the tree builder architecture. If + you wrote your own tree builder and didn't tell me, you'll need to + modify your prepare_markup() method. + +* The UnicodeDammit code that makes guesses at encodings has been + split into its own class, EncodingDetector. A lot of apparently + redundant code has been removed from Unicode, Dammit, and some + undocumented features have also been removed. + +* Beautiful Soup will issue a warning if instead of markup you pass it + a URL or the name of a file on disk (a common beginner's mistake). + +* A number of optimizations improve the performance of the lxml tree + builder by about 33%, the html.parser tree builder by about 20%, and + the html5lib tree builder by about 15%. + +* All find_all calls should now return a ResultSet object. Patch by + Aaron DeVore. [bug=1194034] + += 4.2.1 (20130531) = + +* The default XML formatter will now replace ampersands even if they + appear to be part of entities. That is, "<" will become + "&lt;". The old code was left over from Beautiful Soup 3, which + didn't always turn entities into Unicode characters. + + If you really want the old behavior (maybe because you add new + strings to the tree, those strings include entities, and you want + the formatter to leave them alone on output), it can be found in + EntitySubstitution.substitute_xml_containing_entities(). [bug=1182183] + +* Gave new_string() the ability to create subclasses of + NavigableString. [bug=1181986] + +* Fixed another bug by which the html5lib tree builder could create a + disconnected tree. [bug=1182089] + +* The .previous_element of a BeautifulSoup object is now always None, + not the last element to be parsed. [bug=1182089] + +* Fixed test failures when lxml is not installed. [bug=1181589] + +* html5lib now supports Python 3. Fixed some Python 2-specific + code in the html5lib test suite. [bug=1181624] + +* The html.parser treebuilder can now handle numeric attributes in + text when the hexidecimal name of the attribute starts with a + capital X. Patch by Tim Shirley. [bug=1186242] + += 4.2.0 (20130514) = + +* The Tag.select() method now supports a much wider variety of CSS + selectors. + + - Added support for the adjacent sibling combinator (+) and the + general sibling combinator (~). Tests by "liquider". [bug=1082144] + + - The combinators (>, +, and ~) can now combine with any supported + selector, not just one that selects based on tag name. + + - Added limited support for the "nth-of-type" pseudo-class. Code + by Sven Slootweg. [bug=1109952] + +* The BeautifulSoup class is now aliased to "_s" and "_soup", making + it quicker to type the import statement in an interactive session: + + from bs4 import _s + or + from bs4 import _soup + + The alias may change in the future, so don't use this in code you're + going to run more than once. + +* Added the 'diagnose' submodule, which includes several useful + functions for reporting problems and doing tech support. + + - diagnose(data) tries the given markup on every installed parser, + reporting exceptions and displaying successes. If a parser is not + installed, diagnose() mentions this fact. + + - lxml_trace(data, html=True) runs the given markup through lxml's + XML parser or HTML parser, and prints out the parser events as + they happen. This helps you quickly determine whether a given + problem occurs in lxml code or Beautiful Soup code. + + - htmlparser_trace(data) is the same thing, but for Python's + built-in HTMLParser class. + +* In an HTML document, the contents of a <script> or <style> tag will + no longer undergo entity substitution by default. XML documents work + the same way they did before. [bug=1085953] + +* Methods like get_text() and properties like .strings now only give + you strings that are visible in the document--no comments or + processing commands. [bug=1050164] + +* The prettify() method now leaves the contents of <pre> tags + alone. [bug=1095654] + +* Fix a bug in the html5lib treebuilder which sometimes created + disconnected trees. [bug=1039527] + +* Fix a bug in the lxml treebuilder which crashed when a tag included + an attribute from the predefined "xml:" namespace. [bug=1065617] + +* Fix a bug by which keyword arguments to find_parent() were not + being passed on. [bug=1126734] + +* Stop a crash when unwisely messing with a tag that's been + decomposed. [bug=1097699] + +* Now that lxml's segfault on invalid doctype has been fixed, fixed a + corresponding problem on the Beautiful Soup end that was previously + invisible. [bug=984936] + +* Fixed an exception when an overspecified CSS selector didn't match + anything. Code by Stefaan Lippens. [bug=1168167] + += 4.1.3 (20120820) = + +* Skipped a test under Python 2.6 and Python 3.1 to avoid a spurious + test failure caused by the lousy HTMLParser in those + versions. [bug=1038503] + +* Raise a more specific error (FeatureNotFound) when a requested + parser or parser feature is not installed. Raise NotImplementedError + instead of ValueError when the user calls insert_before() or + insert_after() on the BeautifulSoup object itself. Patch by Aaron + Devore. [bug=1038301] + += 4.1.2 (20120817) = + +* As per PEP-8, allow searching by CSS class using the 'class_' + keyword argument. [bug=1037624] + +* Display namespace prefixes for namespaced attribute names, instead of + the fully-qualified names given by the lxml parser. [bug=1037597] + +* Fixed a crash on encoding when an attribute name contained + non-ASCII characters. + +* When sniffing encodings, if the cchardet library is installed, + Beautiful Soup uses it instead of chardet. cchardet is much + faster. [bug=1020748] + +* Use logging.warning() instead of warning.warn() to notify the user + that characters were replaced with REPLACEMENT + CHARACTER. [bug=1013862] + += 4.1.1 (20120703) = + +* Fixed an html5lib tree builder crash which happened when html5lib + moved a tag with a multivalued attribute from one part of the tree + to another. [bug=1019603] + +* Correctly display closing tags with an XML namespace declared. Patch + by Andreas Kostyrka. [bug=1019635] + +* Fixed a typo that made parsing significantly slower than it should + have been, and also waited too long to close tags with XML + namespaces. [bug=1020268] + +* get_text() now returns an empty Unicode string if there is no text, + rather than an empty bytestring. [bug=1020387] + += 4.1.0 (20120529) = + +* Added experimental support for fixing Windows-1252 characters + embedded in UTF-8 documents. (UnicodeDammit.detwingle()) + +* Fixed the handling of " with the built-in parser. [bug=993871] + +* Comments, processing instructions, document type declarations, and + markup declarations are now treated as preformatted strings, the way + CData blocks are. [bug=1001025] + +* Fixed a bug with the lxml treebuilder that prevented the user from + adding attributes to a tag that didn't originally have + attributes. [bug=1002378] Thanks to Oliver Beattie for the patch. + +* Fixed some edge-case bugs having to do with inserting an element + into a tag it's already inside, and replacing one of a tag's + children with another. [bug=997529] + +* Added the ability to search for attribute values specified in UTF-8. [bug=1003974] + + This caused a major refactoring of the search code. All the tests + pass, but it's possible that some searches will behave differently. + += 4.0.5 (20120427) = + +* Added a new method, wrap(), which wraps an element in a tag. + +* Renamed replace_with_children() to unwrap(), which is easier to + understand and also the jQuery name of the function. + +* Made encoding substitution in <meta> tags completely transparent (no + more %SOUP-ENCODING%). + +* Fixed a bug in decoding data that contained a byte-order mark, such + as data encoded in UTF-16LE. [bug=988980] + +* Fixed a bug that made the HTMLParser treebuilder generate XML + definitions ending with two question marks instead of + one. [bug=984258] + +* Upon document generation, CData objects are no longer run through + the formatter. [bug=988905] + +* The test suite now passes when lxml is not installed, whether or not + html5lib is installed. [bug=987004] + +* Print a warning on HTMLParseErrors to let people know they should + install a better parser library. + += 4.0.4 (20120416) = + +* Fixed a bug that sometimes created disconnected trees. + +* Fixed a bug with the string setter that moved a string around the + tree instead of copying it. [bug=983050] + +* Attribute values are now run through the provided output formatter. + Previously they were always run through the 'minimal' formatter. In + the future I may make it possible to specify different formatters + for attribute values and strings, but for now, consistent behavior + is better than inconsistent behavior. [bug=980237] + +* Added the missing renderContents method from Beautiful Soup 3. Also + added an encode_contents() method to go along with decode_contents(). + +* Give a more useful error when the user tries to run the Python 2 + version of BS under Python 3. + +* UnicodeDammit can now convert Microsoft smart quotes to ASCII with + UnicodeDammit(markup, smart_quotes_to="ascii"). + += 4.0.3 (20120403) = + +* Fixed a typo that caused some versions of Python 3 to convert the + Beautiful Soup codebase incorrectly. + +* Got rid of the 4.0.2 workaround for HTML documents--it was + unnecessary and the workaround was triggering a (possibly different, + but related) bug in lxml. [bug=972466] + += 4.0.2 (20120326) = + +* Worked around a possible bug in lxml that prevents non-tiny XML + documents from being parsed. [bug=963880, bug=963936] + +* Fixed a bug where specifying `text` while also searching for a tag + only worked if `text` wanted an exact string match. [bug=955942] + += 4.0.1 (20120314) = + +* This is the first official release of Beautiful Soup 4. There is no + 4.0.0 release, to eliminate any possibility that packaging software + might treat "4.0.0" as being an earlier version than "4.0.0b10". + +* Brought BS up to date with the latest release of soupselect, adding + CSS selector support for direct descendant matches and multiple CSS + class matches. + += 4.0.0b10 (20120302) = + +* Added support for simple CSS selectors, taken from the soupselect project. + +* Fixed a crash when using html5lib. [bug=943246] + +* In HTML5-style <meta charset="foo"> tags, the value of the "charset" + attribute is now replaced with the appropriate encoding on + output. [bug=942714] + +* Fixed a bug that caused calling a tag to sometimes call find_all() + with the wrong arguments. [bug=944426] + +* For backwards compatibility, brought back the BeautifulStoneSoup + class as a deprecated wrapper around BeautifulSoup. + += 4.0.0b9 (20120228) = + +* Fixed the string representation of DOCTYPEs that have both a public + ID and a system ID. + +* Fixed the generated XML declaration. + +* Renamed Tag.nsprefix to Tag.prefix, for consistency with + NamespacedAttribute. + +* Fixed a test failure that occured on Python 3.x when chardet was + installed. + +* Made prettify() return Unicode by default, so it will look nice on + Python 3 when passed into print(). + += 4.0.0b8 (20120224) = + +* All tree builders now preserve namespace information in the + documents they parse. If you use the html5lib parser or lxml's XML + parser, you can access the namespace URL for a tag as tag.namespace. + + However, there is no special support for namespace-oriented + searching or tree manipulation. When you search the tree, you need + to use namespace prefixes exactly as they're used in the original + document. + +* The string representation of a DOCTYPE always ends in a newline. + +* Issue a warning if the user tries to use a SoupStrainer in + conjunction with the html5lib tree builder, which doesn't support + them. + += 4.0.0b7 (20120223) = + +* Upon decoding to string, any characters that can't be represented in + your chosen encoding will be converted into numeric XML entity + references. + +* Issue a warning if characters were replaced with REPLACEMENT + CHARACTER during Unicode conversion. + +* Restored compatibility with Python 2.6. + +* The install process no longer installs docs or auxillary text files. + +* It's now possible to deepcopy a BeautifulSoup object created with + Python's built-in HTML parser. + +* About 100 unit tests that "test" the behavior of various parsers on + invalid markup have been removed. Legitimate changes to those + parsers caused these tests to fail, indicating that perhaps + Beautiful Soup should not test the behavior of foreign + libraries. + + The problematic unit tests have been reformulated as informational + comparisons generated by the script + scripts/demonstrate_parser_differences.py. + + This makes Beautiful Soup compatible with html5lib version 0.95 and + future versions of HTMLParser. + += 4.0.0b6 (20120216) = + +* Multi-valued attributes like "class" always have a list of values, + even if there's only one value in the list. + +* Added a number of multi-valued attributes defined in HTML5. + +* Stopped generating a space before the slash that closes an + empty-element tag. This may come back if I add a special XHTML mode + (http://www.w3.org/TR/xhtml1/#C_2), but right now it's pretty + useless. + +* Passing text along with tag-specific arguments to a find* method: + + find("a", text="Click here") + + will find tags that contain the given text as their + .string. Previously, the tag-specific arguments were ignored and + only strings were searched. + +* Fixed a bug that caused the html5lib tree builder to build a + partially disconnected tree. Generally cleaned up the html5lib tree + builder. + +* If you restrict a multi-valued attribute like "class" to a string + that contains spaces, Beautiful Soup will only consider it a match + if the values correspond to that specific string. + += 4.0.0b5 (20120209) = + +* Rationalized Beautiful Soup's treatment of CSS class. A tag + belonging to multiple CSS classes is treated as having a list of + values for the 'class' attribute. Searching for a CSS class will + match *any* of the CSS classes. + + This actually affects all attributes that the HTML standard defines + as taking multiple values (class, rel, rev, archive, accept-charset, + and headers), but 'class' is by far the most common. [bug=41034] + +* If you pass anything other than a dictionary as the second argument + to one of the find* methods, it'll assume you want to use that + object to search against a tag's CSS classes. Previously this only + worked if you passed in a string. + +* Fixed a bug that caused a crash when you passed a dictionary as an + attribute value (possibly because you mistyped "attrs"). [bug=842419] + +* Unicode, Dammit now detects the encoding in HTML 5-style <meta> tags + like <meta charset="utf-8" />. [bug=837268] + +* If Unicode, Dammit can't figure out a consistent encoding for a + page, it will try each of its guesses again, with errors="replace" + instead of errors="strict". This may mean that some data gets + replaced with REPLACEMENT CHARACTER, but at least most of it will + get turned into Unicode. [bug=754903] + +* Patched over a bug in html5lib (?) that was crashing Beautiful Soup + on certain kinds of markup. [bug=838800] + +* Fixed a bug that wrecked the tree if you replaced an element with an + empty string. [bug=728697] + +* Improved Unicode, Dammit's behavior when you give it Unicode to + begin with. + += 4.0.0b4 (20120208) = + +* Added BeautifulSoup.new_string() to go along with BeautifulSoup.new_tag() + +* BeautifulSoup.new_tag() will follow the rules of whatever + tree-builder was used to create the original BeautifulSoup object. A + new <p> tag will look like "<p />" if the soup object was created to + parse XML, but it will look like "<p></p>" if the soup object was + created to parse HTML. + +* We pass in strict=False to html.parser on Python 3, greatly + improving html.parser's ability to handle bad HTML. + +* We also monkeypatch a serious bug in html.parser that made + strict=False disastrous on Python 3.2.2. + +* Replaced the "substitute_html_entities" argument with the + more general "formatter" argument. + +* Bare ampersands and angle brackets are always converted to XML + entities unless the user prevents it. + +* Added PageElement.insert_before() and PageElement.insert_after(), + which let you put an element into the parse tree with respect to + some other element. + +* Raise an exception when the user tries to do something nonsensical + like insert a tag into itself. + + += 4.0.0b3 (20120203) = + +Beautiful Soup 4 is a nearly-complete rewrite that removes Beautiful +Soup's custom HTML parser in favor of a system that lets you write a +little glue code and plug in any HTML or XML parser you want. + +Beautiful Soup 4.0 comes with glue code for four parsers: + + * Python's standard HTMLParser (html.parser in Python 3) + * lxml's HTML and XML parsers + * html5lib's HTML parser + +HTMLParser is the default, but I recommend you install lxml if you +can. + +For complete documentation, see the Sphinx documentation in +bs4/doc/source/. What follows is a summary of the changes from +Beautiful Soup 3. + +=== The module name has changed === + +Previously you imported the BeautifulSoup class from a module also +called BeautifulSoup. To save keystrokes and make it clear which +version of the API is in use, the module is now called 'bs4': + + >>> from bs4 import BeautifulSoup + +=== It works with Python 3 === + +Beautiful Soup 3.1.0 worked with Python 3, but the parser it used was +so bad that it barely worked at all. Beautiful Soup 4 works with +Python 3, and since its parser is pluggable, you don't sacrifice +quality. + +Special thanks to Thomas Kluyver and Ezio Melotti for getting Python 3 +support to the finish line. Ezio Melotti is also to thank for greatly +improving the HTML parser that comes with Python 3.2. + +=== CDATA sections are normal text, if they're understood at all. === + +Currently, the lxml and html5lib HTML parsers ignore CDATA sections in +markup: + + <p><![CDATA[foo]]></p> => <p></p> + +A future version of html5lib will turn CDATA sections into text nodes, +but only within tags like <svg> and <math>: + + <svg><![CDATA[foo]]></svg> => <p>foo</p> + +The default XML parser (which uses lxml behind the scenes) turns CDATA +sections into ordinary text elements: + + <p><![CDATA[foo]]></p> => <p>foo</p> + +In theory it's possible to preserve the CDATA sections when using the +XML parser, but I don't see how to get it to work in practice. + +=== Miscellaneous other stuff === + +If the BeautifulSoup instance has .is_xml set to True, an appropriate +XML declaration will be emitted when the tree is transformed into a +string: + + <?xml version="1.0" encoding="utf-8"> + <markup> + ... + </markup> + +The ['lxml', 'xml'] tree builder sets .is_xml to True; the other tree +builders set it to False. If you want to parse XHTML with an HTML +parser, you can set it manually. + + += 3.2.0 = + +The 3.1 series wasn't very useful, so I renamed the 3.0 series to 3.2 +to make it obvious which one you should use. + += 3.1.0 = + +A hybrid version that supports 2.4 and can be automatically converted +to run under Python 3.0. There are three backwards-incompatible +changes you should be aware of, but no new features or deliberate +behavior changes. + +1. str() may no longer do what you want. This is because the meaning +of str() inverts between Python 2 and 3; in Python 2 it gives you a +byte string, in Python 3 it gives you a Unicode string. + +The effect of this is that you can't pass an encoding to .__str__ +anymore. Use encode() to get a string and decode() to get Unicode, and +you'll be ready (well, readier) for Python 3. + +2. Beautiful Soup is now based on HTMLParser rather than SGMLParser, +which is gone in Python 3. There's some bad HTML that SGMLParser +handled but HTMLParser doesn't, usually to do with attribute values +that aren't closed or have brackets inside them: + + <a href="foo</a>, </a><a href="bar">baz</a> + <a b="<a>">', '<a b="<a>"></a><a>"></a> + +A later version of Beautiful Soup will allow you to plug in different +parsers to make tradeoffs between speed and the ability to handle bad +HTML. + +3. In Python 3 (but not Python 2), HTMLParser converts entities within +attributes to the corresponding Unicode characters. In Python 2 it's +possible to parse this string and leave the é intact. + + <a href="http://crummy.com?sacré&bleu"> + +In Python 3, the é is always converted to \xe9 during +parsing. + + += 3.0.7a = + +Added an import that makes BS work in Python 2.3. + + += 3.0.7 = + +Fixed a UnicodeDecodeError when unpickling documents that contain +non-ASCII characters. + +Fixed a TypeError that occured in some circumstances when a tag +contained no text. + +Jump through hoops to avoid the use of chardet, which can be extremely +slow in some circumstances. UTF-8 documents should never trigger the +use of chardet. + +Whitespace is preserved inside <pre> and <textarea> tags that contain +nothing but whitespace. + +Beautiful Soup can now parse a doctype that's scoped to an XML namespace. + + += 3.0.6 = + +Got rid of a very old debug line that prevented chardet from working. + +Added a Tag.decompose() method that completely disconnects a tree or a +subset of a tree, breaking it up into bite-sized pieces that are +easy for the garbage collecter to collect. + +Tag.extract() now returns the tag that was extracted. + +Tag.findNext() now does something with the keyword arguments you pass +it instead of dropping them on the floor. + +Fixed a Unicode conversion bug. + +Fixed a bug that garbled some <meta> tags when rewriting them. + + += 3.0.5 = + +Soup objects can now be pickled, and copied with copy.deepcopy. + +Tag.append now works properly on existing BS objects. (It wasn't +originally intended for outside use, but it can be now.) (Giles +Radford) + +Passing in a nonexistent encoding will no longer crash the parser on +Python 2.4 (John Nagle). + +Fixed an underlying bug in SGMLParser that thinks ASCII has 255 +characters instead of 127 (John Nagle). + +Entities are converted more consistently to Unicode characters. + +Entity references in attribute values are now converted to Unicode +characters when appropriate. Numeric entities are always converted, +because SGMLParser always converts them outside of attribute values. + +ALL_ENTITIES happens to just be the XHTML entities, so I renamed it to +XHTML_ENTITIES. + +The regular expression for bare ampersands was too loose. In some +cases ampersands were not being escaped. (Sam Ruby?) + +Non-breaking spaces and other special Unicode space characters are no +longer folded to ASCII spaces. (Robert Leftwich) + +Information inside a TEXTAREA tag is now parsed literally, not as HTML +tags. TEXTAREA now works exactly the same way as SCRIPT. (Zephyr Fang) + += 3.0.4 = + +Fixed a bug that crashed Unicode conversion in some cases. + +Fixed a bug that prevented UnicodeDammit from being used as a +general-purpose data scrubber. + +Fixed some unit test failures when running against Python 2.5. + +When considering whether to convert smart quotes, UnicodeDammit now +looks at the original encoding in a case-insensitive way. + += 3.0.3 (20060606) = + +Beautiful Soup is now usable as a way to clean up invalid XML/HTML (be +sure to pass in an appropriate value for convertEntities, or XML/HTML +entities might stick around that aren't valid in HTML/XML). The result +may not validate, but it should be good enough to not choke a +real-world XML parser. Specifically, the output of a properly +constructed soup object should always be valid as part of an XML +document, but parts may be missing if they were missing in the +original. As always, if the input is valid XML, the output will also +be valid. + += 3.0.2 (20060602) = + +Previously, Beautiful Soup correctly handled attribute values that +contained embedded quotes (sometimes by escaping), but not other kinds +of XML character. Now, it correctly handles or escapes all special XML +characters in attribute values. + +I aliased methods to the 2.x names (fetch, find, findText, etc.) for +backwards compatibility purposes. Those names are deprecated and if I +ever do a 4.0 I will remove them. I will, I tell you! + +Fixed a bug where the findAll method wasn't passing along any keyword +arguments. + +When run from the command line, Beautiful Soup now acts as an HTML +pretty-printer, not an XML pretty-printer. + += 3.0.1 (20060530) = + +Reintroduced the "fetch by CSS class" shortcut. I thought keyword +arguments would replace it, but they don't. You can't call soup('a', +class='foo') because class is a Python keyword. + +If Beautiful Soup encounters a meta tag that declares the encoding, +but a SoupStrainer tells it not to parse that tag, Beautiful Soup will +no longer try to rewrite the meta tag to mention the new +encoding. Basically, this makes SoupStrainers work in real-world +applications instead of crashing the parser. + += 3.0.0 "Who would not give all else for two p" (20060528) = + +This release is not backward-compatible with previous releases. If +you've got code written with a previous version of the library, go +ahead and keep using it, unless one of the features mentioned here +really makes your life easier. Since the library is self-contained, +you can include an old copy of the library in your old applications, +and use the new version for everything else. + +The documentation has been rewritten and greatly expanded with many +more examples. + +Beautiful Soup autodetects the encoding of a document (or uses the one +you specify), and converts it from its native encoding to +Unicode. Internally, it only deals with Unicode strings. When you +print out the document, it converts to UTF-8 (or another encoding you +specify). [Doc reference] + +It's now easy to make large-scale changes to the parse tree without +screwing up the navigation members. The methods are extract, +replaceWith, and insert. [Doc reference. See also Improving Memory +Usage with extract] + +Passing True in as an attribute value gives you tags that have any +value for that attribute. You don't have to create a regular +expression. Passing None for an attribute value gives you tags that +don't have that attribute at all. + +Tag objects now know whether or not they're self-closing. This avoids +the problem where Beautiful Soup thought that tags like <BR /> were +self-closing even in XML documents. You can customize the self-closing +tags for a parser object by passing them in as a list of +selfClosingTags: you don't have to subclass anymore. + +There's a new built-in parser, MinimalSoup, which has most of +BeautifulSoup's HTML-specific rules, but no tag nesting rules. [Doc +reference] + +You can use a SoupStrainer to tell Beautiful Soup to parse only part +of a document. This saves time and memory, often making Beautiful Soup +about as fast as a custom-built SGMLParser subclass. [Doc reference, +SoupStrainer reference] + +You can (usually) use keyword arguments instead of passing a +dictionary of attributes to a search method. That is, you can replace +soup(args={"id" : "5"}) with soup(id="5"). You can still use args if +(for instance) you need to find an attribute whose name clashes with +the name of an argument to findAll. [Doc reference: **kwargs attrs] + +The method names have changed to the better method names used in +Rubyful Soup. Instead of find methods and fetch methods, there are +only find methods. Instead of a scheme where you can't remember which +method finds one element and which one finds them all, we have find +and findAll. In general, if the method name mentions All or a plural +noun (eg. findNextSiblings), then it finds many elements +method. Otherwise, it only finds one element. [Doc reference] + +Some of the argument names have been renamed for clarity. For instance +avoidParserProblems is now parserMassage. + +Beautiful Soup no longer implements a feed method. You need to pass a +string or a filehandle into the soup constructor, not with feed after +the soup has been created. There is still a feed method, but it's the +feed method implemented by SGMLParser and calling it will bypass +Beautiful Soup and cause problems. + +The NavigableText class has been renamed to NavigableString. There is +no NavigableUnicodeString anymore, because every string inside a +Beautiful Soup parse tree is a Unicode string. + +findText and fetchText are gone. Just pass a text argument into find +or findAll. + +Null was more trouble than it was worth, so I got rid of it. Anything +that used to return Null now returns None. + +Special XML constructs like comments and CDATA now have their own +NavigableString subclasses, instead of being treated as oddly-formed +data. If you parse a document that contains CDATA and write it back +out, the CDATA will still be there. + +When you're parsing a document, you can get Beautiful Soup to convert +XML or HTML entities into the corresponding Unicode characters. [Doc +reference] + += 2.1.1 (20050918) = + +Fixed a serious performance bug in BeautifulStoneSoup which was +causing parsing to be incredibly slow. + +Corrected several entities that were previously being incorrectly +translated from Microsoft smart-quote-like characters. + +Fixed a bug that was breaking text fetch. + +Fixed a bug that crashed the parser when text chunks that look like +HTML tag names showed up within a SCRIPT tag. + +THEAD, TBODY, and TFOOT tags are now nestable within TABLE +tags. Nested tables should parse more sensibly now. + +BASE is now considered a self-closing tag. + += 2.1.0 "Game, or any other dish?" (20050504) = + +Added a wide variety of new search methods which, given a starting +point inside the tree, follow a particular navigation member (like +nextSibling) over and over again, looking for Tag and NavigableText +objects that match certain criteria. The new methods are findNext, +fetchNext, findPrevious, fetchPrevious, findNextSibling, +fetchNextSiblings, findPreviousSibling, fetchPreviousSiblings, +findParent, and fetchParents. All of these use the same basic code +used by first and fetch, so you can pass your weird ways of matching +things into these methods. + +The fetch method and its derivatives now accept a limit argument. + +You can now pass keyword arguments when calling a Tag object as though +it were a method. + +Fixed a bug that caused all hand-created tags to share a single set of +attributes. + += 2.0.3 (20050501) = + +Fixed Python 2.2 support for iterators. + +Fixed a bug that gave the wrong representation to tags within quote +tags like <script>. + +Took some code from Mark Pilgrim that treats CDATA declarations as +data instead of ignoring them. + +Beautiful Soup's setup.py will now do an install even if the unit +tests fail. It won't build a source distribution if the unit tests +fail, so I can't release a new version unless they pass. + += 2.0.2 (20050416) = + +Added the unit tests in a separate module, and packaged it with +distutils. + +Fixed a bug that sometimes caused renderContents() to return a Unicode +string even if there was no Unicode in the original string. + +Added the done() method, which closes all of the parser's open +tags. It gets called automatically when you pass in some text to the +constructor of a parser class; otherwise you must call it yourself. + +Reinstated some backwards compatibility with 1.x versions: referencing +the string member of a NavigableText object returns the NavigableText +object instead of throwing an error. + += 2.0.1 (20050412) = + +Fixed a bug that caused bad results when you tried to reference a tag +name shorter than 3 characters as a member of a Tag, eg. tag.table.td. + +Made sure all Tags have the 'hidden' attribute so that an attempt to +access tag.hidden doesn't spawn an attempt to find a tag named +'hidden'. + +Fixed a bug in the comparison operator. + += 2.0.0 "Who cares for fish?" (20050410) + +Beautiful Soup version 1 was very useful but also pretty stupid. I +originally wrote it without noticing any of the problems inherent in +trying to build a parse tree out of ambiguous HTML tags. This version +solves all of those problems to my satisfaction. It also adds many new +clever things to make up for the removal of the stupid things. + +== Parsing == + +The parser logic has been greatly improved, and the BeautifulSoup +class should much more reliably yield a parse tree that looks like +what the page author intended. For a particular class of odd edge +cases that now causes problems, there is a new class, +ICantBelieveItsBeautifulSoup. + +By default, Beautiful Soup now performs some cleanup operations on +text before parsing it. This is to avoid common problems with bad +definitions and self-closing tags that crash SGMLParser. You can +provide your own set of cleanup operations, or turn it off +altogether. The cleanup operations include fixing self-closing tags +that don't close, and replacing Microsoft smart quotes and similar +characters with their HTML entity equivalents. + +You can now get a pretty-print version of parsed HTML to get a visual +picture of how Beautiful Soup parses it, with the Tag.prettify() +method. + +== Strings and Unicode == + +There are separate NavigableText subclasses for ASCII and Unicode +strings. These classes directly subclass the corresponding base data +types. This means you can treat NavigableText objects as strings +instead of having to call methods on them to get the strings. + +str() on a Tag always returns a string, and unicode() always returns +Unicode. Previously it was inconsistent. + +== Tree traversal == + +In a first() or fetch() call, the tag name or the desired value of an +attribute can now be any of the following: + + * A string (matches that specific tag or that specific attribute value) + * A list of strings (matches any tag or attribute value in the list) + * A compiled regular expression object (matches any tag or attribute + value that matches the regular expression) + * A callable object that takes the Tag object or attribute value as a + string. It returns None/false/empty string if the given string + doesn't match, and any other value if it does. + +This is much easier to use than SQL-style wildcards (see, regular +expressions are good for something). Because of this, I took out +SQL-style wildcards. I'll put them back if someone complains, but +their removal simplifies the code a lot. + +You can use fetch() and first() to search for text in the parse tree, +not just tags. There are new alias methods fetchText() and firstText() +designed for this purpose. As with searching for tags, you can pass in +a string, a regular expression object, or a method to match your text. + +If you pass in something besides a map to the attrs argument of +fetch() or first(), Beautiful Soup will assume you want to match that +thing against the "class" attribute. When you're scraping +well-structured HTML, this makes your code a lot cleaner. + +1.x and 2.x both let you call a Tag object as a shorthand for +fetch(). For instance, foo("bar") is a shorthand for +foo.fetch("bar"). In 2.x, you can also access a specially-named member +of a Tag object as a shorthand for first(). For instance, foo.barTag +is a shorthand for foo.first("bar"). By chaining these shortcuts you +traverse a tree in very little code: for header in +soup.bodyTag.pTag.tableTag('th'): + +If an element relationship (like parent or next) doesn't apply to a +tag, it'll now show up Null instead of None. first() will also return +Null if you ask it for a nonexistent tag. Null is an object that's +just like None, except you can do whatever you want to it and it'll +give you Null instead of throwing an error. + +This lets you do tree traversals like soup.htmlTag.headTag.titleTag +without having to worry if the intermediate stages are actually +there. Previously, if there was no 'head' tag in the document, headTag +in that instance would have been None, and accessing its 'titleTag' +member would have thrown an AttributeError. Now, you can get what you +want when it exists, and get Null when it doesn't, without having to +do a lot of conditionals checking to see if every stage is None. + +There are two new relations between page elements: previousSibling and +nextSibling. They reference the previous and next element at the same +level of the parse tree. For instance, if you have HTML like this: + + <p><ul><li>Foo<br /><li>Bar</ul> + +The first 'li' tag has a previousSibling of Null and its nextSibling +is the second 'li' tag. The second 'li' tag has a nextSibling of Null +and its previousSibling is the first 'li' tag. The previousSibling of +the 'ul' tag is the first 'p' tag. The nextSibling of 'Foo' is the +'br' tag. + +I took out the ability to use fetch() to find tags that have a +specific list of contents. See, I can't even explain it well. It was +really difficult to use, I never used it, and I don't think anyone +else ever used it. To the extent anyone did, they can probably use +fetchText() instead. If it turns out someone needs it I'll think of +another solution. + +== Tree manipulation == + +You can add new attributes to a tag, and delete attributes from a +tag. In 1.x you could only change a tag's existing attributes. + +== Porting Considerations == + +There are three changes in 2.0 that break old code: + +In the post-1.2 release you could pass in a function into fetch(). The +function took a string, the tag name. In 2.0, the function takes the +actual Tag object. + +It's no longer to pass in SQL-style wildcards to fetch(). Use a +regular expression instead. + +The different parsing algorithm means the parse tree may not be shaped +like you expect. This will only actually affect you if your code uses +one of the affected parts. I haven't run into this problem yet while +porting my code. + += Between 1.2 and 2.0 = + +This is the release to get if you want Python 1.5 compatibility. + +The desired value of an attribute can now be any of the following: + + * A string + * A string with SQL-style wildcards + * A compiled RE object + * A callable that returns None/false/empty string if the given value + doesn't match, and any other value otherwise. + +This is much easier to use than SQL-style wildcards (see, regular +expressions are good for something). Because of this, I no longer +recommend you use SQL-style wildcards. They may go away in a future +release to clean up the code. + +Made Beautiful Soup handle processing instructions as text instead of +ignoring them. + +Applied patch from Richie Hindle (richie at entrian dot com) that +makes tag.string a shorthand for tag.contents[0].string when the tag +has only one string-owning child. + +Added still more nestable tags. The nestable tags thing won't work in +a lot of cases and needs to be rethought. + +Fixed an edge case where searching for "%foo" would match any string +shorter than "foo". + += 1.2 "Who for such dainties would not stoop?" (20040708) = + +Applied patch from Ben Last (ben at benlast dot com) that made +Tag.renderContents() correctly handle Unicode. + +Made BeautifulStoneSoup even dumber by making it not implicitly close +a tag when another tag of the same type is encountered; only when an +actual closing tag is encountered. This change courtesy of Fuzzy (mike +at pcblokes dot com). BeautifulSoup still works as before. + += 1.1 "Swimming in a hot tureen" = + +Added more 'nestable' tags. Changed popping semantics so that when a +nestable tag is encountered, tags are popped up to the previously +encountered nestable tag (of whatever kind). I will revert this if +enough people complain, but it should make more people's lives easier +than harder. This enhancement was suggested by Anthony Baxter (anthony +at interlink dot com dot au). + += 1.0 "So rich and green" (20040420) = + +Initial release. diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/__init__.py b/import-layers/yocto-poky/bitbake/lib/bs4/__init__.py new file mode 100644 index 000000000..7ba34269a --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/__init__.py @@ -0,0 +1,406 @@ +"""Beautiful Soup +Elixir and Tonic +"The Screen-Scraper's Friend" +http://www.crummy.com/software/BeautifulSoup/ + +Beautiful Soup uses a pluggable XML or HTML parser to parse a +(possibly invalid) document into a tree representation. Beautiful Soup +provides provides methods and Pythonic idioms that make it easy to +navigate, search, and modify the parse tree. + +Beautiful Soup works with Python 2.6 and up. It works better if lxml +and/or html5lib is installed. + +For more than you ever wanted to know about Beautiful Soup, see the +documentation: +http://www.crummy.com/software/BeautifulSoup/bs4/doc/ +""" + +__author__ = "Leonard Richardson (leonardr@segfault.org)" +__version__ = "4.3.2" +__copyright__ = "Copyright (c) 2004-2013 Leonard Richardson" +__license__ = "MIT" + +__all__ = ['BeautifulSoup'] + +import os +import re +import warnings + +from .builder import builder_registry, ParserRejectedMarkup +from .dammit import UnicodeDammit +from .element import ( + CData, + Comment, + DEFAULT_OUTPUT_ENCODING, + Declaration, + Doctype, + NavigableString, + PageElement, + ProcessingInstruction, + ResultSet, + SoupStrainer, + Tag, + ) + +# The very first thing we do is give a useful error if someone is +# running this code under Python 3 without converting it. +syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).' + +class BeautifulSoup(Tag): + """ + This class defines the basic interface called by the tree builders. + + These methods will be called by the parser: + reset() + feed(markup) + + The tree builder may call these methods from its feed() implementation: + handle_starttag(name, attrs) # See note about return value + handle_endtag(name) + handle_data(data) # Appends to the current data node + endData(containerClass=NavigableString) # Ends the current data node + + No matter how complicated the underlying parser is, you should be + able to build a tree using 'start tag' events, 'end tag' events, + 'data' events, and "done with data" events. + + If you encounter an empty-element tag (aka a self-closing tag, + like HTML's <br> tag), call handle_starttag and then + handle_endtag. + """ + ROOT_TAG_NAME = u'[document]' + + # If the end-user gives no indication which tree builder they + # want, look for one with these features. + DEFAULT_BUILDER_FEATURES = ['html', 'fast'] + + ASCII_SPACES = '\x20\x0a\x09\x0c\x0d' + + def __init__(self, markup="", features=None, builder=None, + parse_only=None, from_encoding=None, **kwargs): + """The Soup object is initialized as the 'root tag', and the + provided markup (which can be a string or a file-like object) + is fed into the underlying parser.""" + + if 'convertEntities' in kwargs: + warnings.warn( + "BS4 does not respect the convertEntities argument to the " + "BeautifulSoup constructor. Entities are always converted " + "to Unicode characters.") + + if 'markupMassage' in kwargs: + del kwargs['markupMassage'] + warnings.warn( + "BS4 does not respect the markupMassage argument to the " + "BeautifulSoup constructor. The tree builder is responsible " + "for any necessary markup massage.") + + if 'smartQuotesTo' in kwargs: + del kwargs['smartQuotesTo'] + warnings.warn( + "BS4 does not respect the smartQuotesTo argument to the " + "BeautifulSoup constructor. Smart quotes are always converted " + "to Unicode characters.") + + if 'selfClosingTags' in kwargs: + del kwargs['selfClosingTags'] + warnings.warn( + "BS4 does not respect the selfClosingTags argument to the " + "BeautifulSoup constructor. The tree builder is responsible " + "for understanding self-closing tags.") + + if 'isHTML' in kwargs: + del kwargs['isHTML'] + warnings.warn( + "BS4 does not respect the isHTML argument to the " + "BeautifulSoup constructor. You can pass in features='html' " + "or features='xml' to get a builder capable of handling " + "one or the other.") + + def deprecated_argument(old_name, new_name): + if old_name in kwargs: + warnings.warn( + 'The "%s" argument to the BeautifulSoup constructor ' + 'has been renamed to "%s."' % (old_name, new_name)) + value = kwargs[old_name] + del kwargs[old_name] + return value + return None + + parse_only = parse_only or deprecated_argument( + "parseOnlyThese", "parse_only") + + from_encoding = from_encoding or deprecated_argument( + "fromEncoding", "from_encoding") + + if len(kwargs) > 0: + arg = kwargs.keys().pop() + raise TypeError( + "__init__() got an unexpected keyword argument '%s'" % arg) + + if builder is None: + if isinstance(features, basestring): + features = [features] + if features is None or len(features) == 0: + features = self.DEFAULT_BUILDER_FEATURES + builder_class = builder_registry.lookup(*features) + if builder_class is None: + raise FeatureNotFound( + "Couldn't find a tree builder with the features you " + "requested: %s. Do you need to install a parser library?" + % ",".join(features)) + builder = builder_class() + self.builder = builder + self.is_xml = builder.is_xml + self.builder.soup = self + + self.parse_only = parse_only + + if hasattr(markup, 'read'): # It's a file-type object. + markup = markup.read() + elif len(markup) <= 256: + # Print out warnings for a couple beginner problems + # involving passing non-markup to Beautiful Soup. + # Beautiful Soup will still parse the input as markup, + # just in case that's what the user really wants. + if (isinstance(markup, unicode) + and not os.path.supports_unicode_filenames): + possible_filename = markup.encode("utf8") + else: + possible_filename = markup + is_file = False + try: + is_file = os.path.exists(possible_filename) + except Exception, e: + # This is almost certainly a problem involving + # characters not valid in filenames on this + # system. Just let it go. + pass + if is_file: + warnings.warn( + '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup) + if markup[:5] == "http:" or markup[:6] == "https:": + # TODO: This is ugly but I couldn't get it to work in + # Python 3 otherwise. + if ((isinstance(markup, bytes) and not b' ' in markup) + or (isinstance(markup, unicode) and not u' ' in markup)): + warnings.warn( + '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup) + + for (self.markup, self.original_encoding, self.declared_html_encoding, + self.contains_replacement_characters) in ( + self.builder.prepare_markup(markup, from_encoding)): + self.reset() + try: + self._feed() + break + except ParserRejectedMarkup: + pass + + # Clear out the markup and remove the builder's circular + # reference to this object. + self.markup = None + self.builder.soup = None + + def _feed(self): + # Convert the document to Unicode. + self.builder.reset() + + self.builder.feed(self.markup) + # Close out any unfinished strings and close all the open tags. + self.endData() + while self.currentTag.name != self.ROOT_TAG_NAME: + self.popTag() + + def reset(self): + Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME) + self.hidden = 1 + self.builder.reset() + self.current_data = [] + self.currentTag = None + self.tagStack = [] + self.preserve_whitespace_tag_stack = [] + self.pushTag(self) + + def new_tag(self, name, namespace=None, nsprefix=None, **attrs): + """Create a new tag associated with this soup.""" + return Tag(None, self.builder, name, namespace, nsprefix, attrs) + + def new_string(self, s, subclass=NavigableString): + """Create a new NavigableString associated with this soup.""" + navigable = subclass(s) + navigable.setup() + return navigable + + def insert_before(self, successor): + raise NotImplementedError("BeautifulSoup objects don't support insert_before().") + + def insert_after(self, successor): + raise NotImplementedError("BeautifulSoup objects don't support insert_after().") + + def popTag(self): + tag = self.tagStack.pop() + if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]: + self.preserve_whitespace_tag_stack.pop() + #print "Pop", tag.name + if self.tagStack: + self.currentTag = self.tagStack[-1] + return self.currentTag + + def pushTag(self, tag): + #print "Push", tag.name + if self.currentTag: + self.currentTag.contents.append(tag) + self.tagStack.append(tag) + self.currentTag = self.tagStack[-1] + if tag.name in self.builder.preserve_whitespace_tags: + self.preserve_whitespace_tag_stack.append(tag) + + def endData(self, containerClass=NavigableString): + if self.current_data: + current_data = u''.join(self.current_data) + # If whitespace is not preserved, and this string contains + # nothing but ASCII spaces, replace it with a single space + # or newline. + if not self.preserve_whitespace_tag_stack: + strippable = True + for i in current_data: + if i not in self.ASCII_SPACES: + strippable = False + break + if strippable: + if '\n' in current_data: + current_data = '\n' + else: + current_data = ' ' + + # Reset the data collector. + self.current_data = [] + + # Should we add this string to the tree at all? + if self.parse_only and len(self.tagStack) <= 1 and \ + (not self.parse_only.text or \ + not self.parse_only.search(current_data)): + return + + o = containerClass(current_data) + self.object_was_parsed(o) + + def object_was_parsed(self, o, parent=None, most_recent_element=None): + """Add an object to the parse tree.""" + parent = parent or self.currentTag + most_recent_element = most_recent_element or self._most_recent_element + o.setup(parent, most_recent_element) + + if most_recent_element is not None: + most_recent_element.next_element = o + self._most_recent_element = o + parent.contents.append(o) + + def _popToTag(self, name, nsprefix=None, inclusivePop=True): + """Pops the tag stack up to and including the most recent + instance of the given tag. If inclusivePop is false, pops the tag + stack up to but *not* including the most recent instqance of + the given tag.""" + #print "Popping to %s" % name + if name == self.ROOT_TAG_NAME: + # The BeautifulSoup object itself can never be popped. + return + + most_recently_popped = None + + stack_size = len(self.tagStack) + for i in range(stack_size - 1, 0, -1): + t = self.tagStack[i] + if (name == t.name and nsprefix == t.prefix): + if inclusivePop: + most_recently_popped = self.popTag() + break + most_recently_popped = self.popTag() + + return most_recently_popped + + def handle_starttag(self, name, namespace, nsprefix, attrs): + """Push a start tag on to the stack. + + If this method returns None, the tag was rejected by the + SoupStrainer. You should proceed as if the tag had not occured + in the document. For instance, if this was a self-closing tag, + don't call handle_endtag. + """ + + # print "Start tag %s: %s" % (name, attrs) + self.endData() + + if (self.parse_only and len(self.tagStack) <= 1 + and (self.parse_only.text + or not self.parse_only.search_tag(name, attrs))): + return None + + tag = Tag(self, self.builder, name, namespace, nsprefix, attrs, + self.currentTag, self._most_recent_element) + if tag is None: + return tag + if self._most_recent_element: + self._most_recent_element.next_element = tag + self._most_recent_element = tag + self.pushTag(tag) + return tag + + def handle_endtag(self, name, nsprefix=None): + #print "End tag: " + name + self.endData() + self._popToTag(name, nsprefix) + + def handle_data(self, data): + self.current_data.append(data) + + def decode(self, pretty_print=False, + eventual_encoding=DEFAULT_OUTPUT_ENCODING, + formatter="minimal"): + """Returns a string or Unicode representation of this document. + To get Unicode, pass None for encoding.""" + + if self.is_xml: + # Print the XML declaration + encoding_part = '' + if eventual_encoding != None: + encoding_part = ' encoding="%s"' % eventual_encoding + prefix = u'<?xml version="1.0"%s?>\n' % encoding_part + else: + prefix = u'' + if not pretty_print: + indent_level = None + else: + indent_level = 0 + return prefix + super(BeautifulSoup, self).decode( + indent_level, eventual_encoding, formatter) + +# Alias to make it easier to type import: 'from bs4 import _soup' +_s = BeautifulSoup +_soup = BeautifulSoup + +class BeautifulStoneSoup(BeautifulSoup): + """Deprecated interface to an XML parser.""" + + def __init__(self, *args, **kwargs): + kwargs['features'] = 'xml' + warnings.warn( + 'The BeautifulStoneSoup class is deprecated. Instead of using ' + 'it, pass features="xml" into the BeautifulSoup constructor.') + super(BeautifulStoneSoup, self).__init__(*args, **kwargs) + + +class StopParsing(Exception): + pass + +class FeatureNotFound(ValueError): + pass + + +#By default, act as an HTML pretty-printer. +if __name__ == '__main__': + import sys + soup = BeautifulSoup(sys.stdin) + print soup.prettify() diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/builder/__init__.py b/import-layers/yocto-poky/bitbake/lib/bs4/builder/__init__.py new file mode 100644 index 000000000..740f5f29c --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/builder/__init__.py @@ -0,0 +1,321 @@ +from collections import defaultdict +import itertools +import sys +from bs4.element import ( + CharsetMetaAttributeValue, + ContentMetaAttributeValue, + whitespace_re + ) + +__all__ = [ + 'HTMLTreeBuilder', + 'SAXTreeBuilder', + 'TreeBuilder', + 'TreeBuilderRegistry', + ] + +# Some useful features for a TreeBuilder to have. +FAST = 'fast' +PERMISSIVE = 'permissive' +STRICT = 'strict' +XML = 'xml' +HTML = 'html' +HTML_5 = 'html5' + + +class TreeBuilderRegistry(object): + + def __init__(self): + self.builders_for_feature = defaultdict(list) + self.builders = [] + + def register(self, treebuilder_class): + """Register a treebuilder based on its advertised features.""" + for feature in treebuilder_class.features: + self.builders_for_feature[feature].insert(0, treebuilder_class) + self.builders.insert(0, treebuilder_class) + + def lookup(self, *features): + if len(self.builders) == 0: + # There are no builders at all. + return None + + if len(features) == 0: + # They didn't ask for any features. Give them the most + # recently registered builder. + return self.builders[0] + + # Go down the list of features in order, and eliminate any builders + # that don't match every feature. + features = list(features) + features.reverse() + candidates = None + candidate_set = None + while len(features) > 0: + feature = features.pop() + we_have_the_feature = self.builders_for_feature.get(feature, []) + if len(we_have_the_feature) > 0: + if candidates is None: + candidates = we_have_the_feature + candidate_set = set(candidates) + else: + # Eliminate any candidates that don't have this feature. + candidate_set = candidate_set.intersection( + set(we_have_the_feature)) + + # The only valid candidates are the ones in candidate_set. + # Go through the original list of candidates and pick the first one + # that's in candidate_set. + if candidate_set is None: + return None + for candidate in candidates: + if candidate in candidate_set: + return candidate + return None + +# The BeautifulSoup class will take feature lists from developers and use them +# to look up builders in this registry. +builder_registry = TreeBuilderRegistry() + +class TreeBuilder(object): + """Turn a document into a Beautiful Soup object tree.""" + + features = [] + + is_xml = False + preserve_whitespace_tags = set() + empty_element_tags = None # A tag will be considered an empty-element + # tag when and only when it has no contents. + + # A value for these tag/attribute combinations is a space- or + # comma-separated list of CDATA, rather than a single CDATA. + cdata_list_attributes = {} + + + def __init__(self): + self.soup = None + + def reset(self): + pass + + def can_be_empty_element(self, tag_name): + """Might a tag with this name be an empty-element tag? + + The final markup may or may not actually present this tag as + self-closing. + + For instance: an HTMLBuilder does not consider a <p> tag to be + an empty-element tag (it's not in + HTMLBuilder.empty_element_tags). This means an empty <p> tag + will be presented as "<p></p>", not "<p />". + + The default implementation has no opinion about which tags are + empty-element tags, so a tag will be presented as an + empty-element tag if and only if it has no contents. + "<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will + be left alone. + """ + if self.empty_element_tags is None: + return True + return tag_name in self.empty_element_tags + + def feed(self, markup): + raise NotImplementedError() + + def prepare_markup(self, markup, user_specified_encoding=None, + document_declared_encoding=None): + return markup, None, None, False + + def test_fragment_to_document(self, fragment): + """Wrap an HTML fragment to make it look like a document. + + Different parsers do this differently. For instance, lxml + introduces an empty <head> tag, and html5lib + doesn't. Abstracting this away lets us write simple tests + which run HTML fragments through the parser and compare the + results against other HTML fragments. + + This method should not be used outside of tests. + """ + return fragment + + def set_up_substitutions(self, tag): + return False + + def _replace_cdata_list_attribute_values(self, tag_name, attrs): + """Replaces class="foo bar" with class=["foo", "bar"] + + Modifies its input in place. + """ + if not attrs: + return attrs + if self.cdata_list_attributes: + universal = self.cdata_list_attributes.get('*', []) + tag_specific = self.cdata_list_attributes.get( + tag_name.lower(), None) + for attr in attrs.keys(): + if attr in universal or (tag_specific and attr in tag_specific): + # We have a "class"-type attribute whose string + # value is a whitespace-separated list of + # values. Split it into a list. + value = attrs[attr] + if isinstance(value, basestring): + values = whitespace_re.split(value) + else: + # html5lib sometimes calls setAttributes twice + # for the same tag when rearranging the parse + # tree. On the second call the attribute value + # here is already a list. If this happens, + # leave the value alone rather than trying to + # split it again. + values = value + attrs[attr] = values + return attrs + +class SAXTreeBuilder(TreeBuilder): + """A Beautiful Soup treebuilder that listens for SAX events.""" + + def feed(self, markup): + raise NotImplementedError() + + def close(self): + pass + + def startElement(self, name, attrs): + attrs = dict((key[1], value) for key, value in list(attrs.items())) + #print "Start %s, %r" % (name, attrs) + self.soup.handle_starttag(name, attrs) + + def endElement(self, name): + #print "End %s" % name + self.soup.handle_endtag(name) + + def startElementNS(self, nsTuple, nodeName, attrs): + # Throw away (ns, nodeName) for now. + self.startElement(nodeName, attrs) + + def endElementNS(self, nsTuple, nodeName): + # Throw away (ns, nodeName) for now. + self.endElement(nodeName) + #handler.endElementNS((ns, node.nodeName), node.nodeName) + + def startPrefixMapping(self, prefix, nodeValue): + # Ignore the prefix for now. + pass + + def endPrefixMapping(self, prefix): + # Ignore the prefix for now. + # handler.endPrefixMapping(prefix) + pass + + def characters(self, content): + self.soup.handle_data(content) + + def startDocument(self): + pass + + def endDocument(self): + pass + + +class HTMLTreeBuilder(TreeBuilder): + """This TreeBuilder knows facts about HTML. + + Such as which tags are empty-element tags. + """ + + preserve_whitespace_tags = set(['pre', 'textarea']) + empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta', + 'spacer', 'link', 'frame', 'base']) + + # The HTML standard defines these attributes as containing a + # space-separated list of values, not a single value. That is, + # class="foo bar" means that the 'class' attribute has two values, + # 'foo' and 'bar', not the single value 'foo bar'. When we + # encounter one of these attributes, we will parse its value into + # a list of values if possible. Upon output, the list will be + # converted back into a string. + cdata_list_attributes = { + "*" : ['class', 'accesskey', 'dropzone'], + "a" : ['rel', 'rev'], + "link" : ['rel', 'rev'], + "td" : ["headers"], + "th" : ["headers"], + "td" : ["headers"], + "form" : ["accept-charset"], + "object" : ["archive"], + + # These are HTML5 specific, as are *.accesskey and *.dropzone above. + "area" : ["rel"], + "icon" : ["sizes"], + "iframe" : ["sandbox"], + "output" : ["for"], + } + + def set_up_substitutions(self, tag): + # We are only interested in <meta> tags + if tag.name != 'meta': + return False + + http_equiv = tag.get('http-equiv') + content = tag.get('content') + charset = tag.get('charset') + + # We are interested in <meta> tags that say what encoding the + # document was originally in. This means HTML 5-style <meta> + # tags that provide the "charset" attribute. It also means + # HTML 4-style <meta> tags that provide the "content" + # attribute and have "http-equiv" set to "content-type". + # + # In both cases we will replace the value of the appropriate + # attribute with a standin object that can take on any + # encoding. + meta_encoding = None + if charset is not None: + # HTML 5 style: + # <meta charset="utf8"> + meta_encoding = charset + tag['charset'] = CharsetMetaAttributeValue(charset) + + elif (content is not None and http_equiv is not None + and http_equiv.lower() == 'content-type'): + # HTML 4 style: + # <meta http-equiv="content-type" content="text/html; charset=utf8"> + tag['content'] = ContentMetaAttributeValue(content) + + return (meta_encoding is not None) + +def register_treebuilders_from(module): + """Copy TreeBuilders from the given module into this module.""" + # I'm fairly sure this is not the best way to do this. + this_module = sys.modules['bs4.builder'] + for name in module.__all__: + obj = getattr(module, name) + + if issubclass(obj, TreeBuilder): + setattr(this_module, name, obj) + this_module.__all__.append(name) + # Register the builder while we're at it. + this_module.builder_registry.register(obj) + +class ParserRejectedMarkup(Exception): + pass + +# Builders are registered in reverse order of priority, so that custom +# builder registrations will take precedence. In general, we want lxml +# to take precedence over html5lib, because it's faster. And we only +# want to use HTMLParser as a last result. +from . import _htmlparser +register_treebuilders_from(_htmlparser) +try: + from . import _html5lib + register_treebuilders_from(_html5lib) +except ImportError: + # They don't have html5lib installed. + pass +try: + from . import _lxml + register_treebuilders_from(_lxml) +except ImportError: + # They don't have lxml installed. + pass diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/builder/_html5lib.py b/import-layers/yocto-poky/bitbake/lib/bs4/builder/_html5lib.py new file mode 100644 index 000000000..7de36ae75 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/builder/_html5lib.py @@ -0,0 +1,285 @@ +__all__ = [ + 'HTML5TreeBuilder', + ] + +import warnings +from bs4.builder import ( + PERMISSIVE, + HTML, + HTML_5, + HTMLTreeBuilder, + ) +from bs4.element import NamespacedAttribute +import html5lib +from html5lib.constants import namespaces +from bs4.element import ( + Comment, + Doctype, + NavigableString, + Tag, + ) + +class HTML5TreeBuilder(HTMLTreeBuilder): + """Use html5lib to build a tree.""" + + features = ['html5lib', PERMISSIVE, HTML_5, HTML] + + def prepare_markup(self, markup, user_specified_encoding): + # Store the user-specified encoding for use later on. + self.user_specified_encoding = user_specified_encoding + yield (markup, None, None, False) + + # These methods are defined by Beautiful Soup. + def feed(self, markup): + if self.soup.parse_only is not None: + warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.") + parser = html5lib.HTMLParser(tree=self.create_treebuilder) + doc = parser.parse(markup, encoding=self.user_specified_encoding) + + # Set the character encoding detected by the tokenizer. + if isinstance(markup, unicode): + # We need to special-case this because html5lib sets + # charEncoding to UTF-8 if it gets Unicode input. + doc.original_encoding = None + else: + doc.original_encoding = parser.tokenizer.stream.charEncoding[0] + + def create_treebuilder(self, namespaceHTMLElements): + self.underlying_builder = TreeBuilderForHtml5lib( + self.soup, namespaceHTMLElements) + return self.underlying_builder + + def test_fragment_to_document(self, fragment): + """See `TreeBuilder`.""" + return u'<html><head></head><body>%s</body></html>' % fragment + + +class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder): + + def __init__(self, soup, namespaceHTMLElements): + self.soup = soup + super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements) + + def documentClass(self): + self.soup.reset() + return Element(self.soup, self.soup, None) + + def insertDoctype(self, token): + name = token["name"] + publicId = token["publicId"] + systemId = token["systemId"] + + doctype = Doctype.for_name_and_ids(name, publicId, systemId) + self.soup.object_was_parsed(doctype) + + def elementClass(self, name, namespace): + tag = self.soup.new_tag(name, namespace) + return Element(tag, self.soup, namespace) + + def commentClass(self, data): + return TextNode(Comment(data), self.soup) + + def fragmentClass(self): + self.soup = BeautifulSoup("") + self.soup.name = "[document_fragment]" + return Element(self.soup, self.soup, None) + + def appendChild(self, node): + # XXX This code is not covered by the BS4 tests. + self.soup.append(node.element) + + def getDocument(self): + return self.soup + + def getFragment(self): + return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element + +class AttrList(object): + def __init__(self, element): + self.element = element + self.attrs = dict(self.element.attrs) + def __iter__(self): + return list(self.attrs.items()).__iter__() + def __setitem__(self, name, value): + "set attr", name, value + self.element[name] = value + def items(self): + return list(self.attrs.items()) + def keys(self): + return list(self.attrs.keys()) + def __len__(self): + return len(self.attrs) + def __getitem__(self, name): + return self.attrs[name] + def __contains__(self, name): + return name in list(self.attrs.keys()) + + +class Element(html5lib.treebuilders._base.Node): + def __init__(self, element, soup, namespace): + html5lib.treebuilders._base.Node.__init__(self, element.name) + self.element = element + self.soup = soup + self.namespace = namespace + + def appendChild(self, node): + string_child = child = None + if isinstance(node, basestring): + # Some other piece of code decided to pass in a string + # instead of creating a TextElement object to contain the + # string. + string_child = child = node + elif isinstance(node, Tag): + # Some other piece of code decided to pass in a Tag + # instead of creating an Element object to contain the + # Tag. + child = node + elif node.element.__class__ == NavigableString: + string_child = child = node.element + else: + child = node.element + + if not isinstance(child, basestring) and child.parent is not None: + node.element.extract() + + if (string_child and self.element.contents + and self.element.contents[-1].__class__ == NavigableString): + # We are appending a string onto another string. + # TODO This has O(n^2) performance, for input like + # "a</a>a</a>a</a>..." + old_element = self.element.contents[-1] + new_element = self.soup.new_string(old_element + string_child) + old_element.replace_with(new_element) + self.soup._most_recent_element = new_element + else: + if isinstance(node, basestring): + # Create a brand new NavigableString from this string. + child = self.soup.new_string(node) + + # Tell Beautiful Soup to act as if it parsed this element + # immediately after the parent's last descendant. (Or + # immediately after the parent, if it has no children.) + if self.element.contents: + most_recent_element = self.element._last_descendant(False) + else: + most_recent_element = self.element + + self.soup.object_was_parsed( + child, parent=self.element, + most_recent_element=most_recent_element) + + def getAttributes(self): + return AttrList(self.element) + + def setAttributes(self, attributes): + if attributes is not None and len(attributes) > 0: + + converted_attributes = [] + for name, value in list(attributes.items()): + if isinstance(name, tuple): + new_name = NamespacedAttribute(*name) + del attributes[name] + attributes[new_name] = value + + self.soup.builder._replace_cdata_list_attribute_values( + self.name, attributes) + for name, value in attributes.items(): + self.element[name] = value + + # The attributes may contain variables that need substitution. + # Call set_up_substitutions manually. + # + # The Tag constructor called this method when the Tag was created, + # but we just set/changed the attributes, so call it again. + self.soup.builder.set_up_substitutions(self.element) + attributes = property(getAttributes, setAttributes) + + def insertText(self, data, insertBefore=None): + if insertBefore: + text = TextNode(self.soup.new_string(data), self.soup) + self.insertBefore(data, insertBefore) + else: + self.appendChild(data) + + def insertBefore(self, node, refNode): + index = self.element.index(refNode.element) + if (node.element.__class__ == NavigableString and self.element.contents + and self.element.contents[index-1].__class__ == NavigableString): + # (See comments in appendChild) + old_node = self.element.contents[index-1] + new_str = self.soup.new_string(old_node + node.element) + old_node.replace_with(new_str) + else: + self.element.insert(index, node.element) + node.parent = self + + def removeChild(self, node): + node.element.extract() + + def reparentChildren(self, new_parent): + """Move all of this tag's children into another tag.""" + element = self.element + new_parent_element = new_parent.element + # Determine what this tag's next_element will be once all the children + # are removed. + final_next_element = element.next_sibling + + new_parents_last_descendant = new_parent_element._last_descendant(False, False) + if len(new_parent_element.contents) > 0: + # The new parent already contains children. We will be + # appending this tag's children to the end. + new_parents_last_child = new_parent_element.contents[-1] + new_parents_last_descendant_next_element = new_parents_last_descendant.next_element + else: + # The new parent contains no children. + new_parents_last_child = None + new_parents_last_descendant_next_element = new_parent_element.next_element + + to_append = element.contents + append_after = new_parent.element.contents + if len(to_append) > 0: + # Set the first child's previous_element and previous_sibling + # to elements within the new parent + first_child = to_append[0] + first_child.previous_element = new_parents_last_descendant + first_child.previous_sibling = new_parents_last_child + + # Fix the last child's next_element and next_sibling + last_child = to_append[-1] + last_child.next_element = new_parents_last_descendant_next_element + last_child.next_sibling = None + + for child in to_append: + child.parent = new_parent_element + new_parent_element.contents.append(child) + + # Now that this element has no children, change its .next_element. + element.contents = [] + element.next_element = final_next_element + + def cloneNode(self): + tag = self.soup.new_tag(self.element.name, self.namespace) + node = Element(tag, self.soup, self.namespace) + for key,value in self.attributes: + node.attributes[key] = value + return node + + def hasContent(self): + return self.element.contents + + def getNameTuple(self): + if self.namespace == None: + return namespaces["html"], self.name + else: + return self.namespace, self.name + + nameTuple = property(getNameTuple) + +class TextNode(Element): + def __init__(self, element, soup): + html5lib.treebuilders._base.Node.__init__(self, None) + self.element = element + self.soup = soup + + def cloneNode(self): + raise NotImplementedError diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/builder/_htmlparser.py b/import-layers/yocto-poky/bitbake/lib/bs4/builder/_htmlparser.py new file mode 100644 index 000000000..ca8d8b892 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/builder/_htmlparser.py @@ -0,0 +1,258 @@ +"""Use the HTMLParser library to parse HTML files that aren't too bad.""" + +__all__ = [ + 'HTMLParserTreeBuilder', + ] + +from HTMLParser import ( + HTMLParser, + HTMLParseError, + ) +import sys +import warnings + +# Starting in Python 3.2, the HTMLParser constructor takes a 'strict' +# argument, which we'd like to set to False. Unfortunately, +# http://bugs.python.org/issue13273 makes strict=True a better bet +# before Python 3.2.3. +# +# At the end of this file, we monkeypatch HTMLParser so that +# strict=True works well on Python 3.2.2. +major, minor, release = sys.version_info[:3] +CONSTRUCTOR_TAKES_STRICT = ( + major > 3 + or (major == 3 and minor > 2) + or (major == 3 and minor == 2 and release >= 3)) + +from bs4.element import ( + CData, + Comment, + Declaration, + Doctype, + ProcessingInstruction, + ) +from bs4.dammit import EntitySubstitution, UnicodeDammit + +from bs4.builder import ( + HTML, + HTMLTreeBuilder, + STRICT, + ) + + +HTMLPARSER = 'html.parser' + +class BeautifulSoupHTMLParser(HTMLParser): + def handle_starttag(self, name, attrs): + # XXX namespace + attr_dict = {} + for key, value in attrs: + # Change None attribute values to the empty string + # for consistency with the other tree builders. + if value is None: + value = '' + attr_dict[key] = value + attrvalue = '""' + self.soup.handle_starttag(name, None, None, attr_dict) + + def handle_endtag(self, name): + self.soup.handle_endtag(name) + + def handle_data(self, data): + self.soup.handle_data(data) + + def handle_charref(self, name): + # XXX workaround for a bug in HTMLParser. Remove this once + # it's fixed. + if name.startswith('x'): + real_name = int(name.lstrip('x'), 16) + elif name.startswith('X'): + real_name = int(name.lstrip('X'), 16) + else: + real_name = int(name) + + try: + data = unichr(real_name) + except (ValueError, OverflowError), e: + data = u"\N{REPLACEMENT CHARACTER}" + + self.handle_data(data) + + def handle_entityref(self, name): + character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name) + if character is not None: + data = character + else: + data = "&%s;" % name + self.handle_data(data) + + def handle_comment(self, data): + self.soup.endData() + self.soup.handle_data(data) + self.soup.endData(Comment) + + def handle_decl(self, data): + self.soup.endData() + if data.startswith("DOCTYPE "): + data = data[len("DOCTYPE "):] + elif data == 'DOCTYPE': + # i.e. "<!DOCTYPE>" + data = '' + self.soup.handle_data(data) + self.soup.endData(Doctype) + + def unknown_decl(self, data): + if data.upper().startswith('CDATA['): + cls = CData + data = data[len('CDATA['):] + else: + cls = Declaration + self.soup.endData() + self.soup.handle_data(data) + self.soup.endData(cls) + + def handle_pi(self, data): + self.soup.endData() + if data.endswith("?") and data.lower().startswith("xml"): + # "An XHTML processing instruction using the trailing '?' + # will cause the '?' to be included in data." - HTMLParser + # docs. + # + # Strip the question mark so we don't end up with two + # question marks. + data = data[:-1] + self.soup.handle_data(data) + self.soup.endData(ProcessingInstruction) + + +class HTMLParserTreeBuilder(HTMLTreeBuilder): + + is_xml = False + features = [HTML, STRICT, HTMLPARSER] + + def __init__(self, *args, **kwargs): + if CONSTRUCTOR_TAKES_STRICT: + kwargs['strict'] = False + self.parser_args = (args, kwargs) + + def prepare_markup(self, markup, user_specified_encoding=None, + document_declared_encoding=None): + """ + :return: A 4-tuple (markup, original encoding, encoding + declared within markup, whether any characters had to be + replaced with REPLACEMENT CHARACTER). + """ + if isinstance(markup, unicode): + yield (markup, None, None, False) + return + + try_encodings = [user_specified_encoding, document_declared_encoding] + dammit = UnicodeDammit(markup, try_encodings, is_html=True) + yield (dammit.markup, dammit.original_encoding, + dammit.declared_html_encoding, + dammit.contains_replacement_characters) + + def feed(self, markup): + args, kwargs = self.parser_args + parser = BeautifulSoupHTMLParser(*args, **kwargs) + parser.soup = self.soup + try: + parser.feed(markup) + except HTMLParseError, e: + warnings.warn(RuntimeWarning( + "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help.")) + raise e + +# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some +# 3.2.3 code. This ensures they don't treat markup like <p></p> as a +# string. +# +# XXX This code can be removed once most Python 3 users are on 3.2.3. +if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT: + import re + attrfind_tolerant = re.compile( + r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*' + r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?') + HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant + + locatestarttagend = re.compile(r""" + <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name + (?:\s+ # whitespace before attribute name + (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name + (?:\s*=\s* # value indicator + (?:'[^']*' # LITA-enclosed value + |\"[^\"]*\" # LIT-enclosed value + |[^'\">\s]+ # bare value + ) + )? + ) + )* + \s* # trailing whitespace +""", re.VERBOSE) + BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend + + from html.parser import tagfind, attrfind + + def parse_starttag(self, i): + self.__starttag_text = None + endpos = self.check_for_whole_start_tag(i) + if endpos < 0: + return endpos + rawdata = self.rawdata + self.__starttag_text = rawdata[i:endpos] + + # Now parse the data between i+1 and j into a tag and attrs + attrs = [] + match = tagfind.match(rawdata, i+1) + assert match, 'unexpected call to parse_starttag()' + k = match.end() + self.lasttag = tag = rawdata[i+1:k].lower() + while k < endpos: + if self.strict: + m = attrfind.match(rawdata, k) + else: + m = attrfind_tolerant.match(rawdata, k) + if not m: + break + attrname, rest, attrvalue = m.group(1, 2, 3) + if not rest: + attrvalue = None + elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ + attrvalue[:1] == '"' == attrvalue[-1:]: + attrvalue = attrvalue[1:-1] + if attrvalue: + attrvalue = self.unescape(attrvalue) + attrs.append((attrname.lower(), attrvalue)) + k = m.end() + + end = rawdata[k:endpos].strip() + if end not in (">", "/>"): + lineno, offset = self.getpos() + if "\n" in self.__starttag_text: + lineno = lineno + self.__starttag_text.count("\n") + offset = len(self.__starttag_text) \ + - self.__starttag_text.rfind("\n") + else: + offset = offset + len(self.__starttag_text) + if self.strict: + self.error("junk characters in start tag: %r" + % (rawdata[k:endpos][:20],)) + self.handle_data(rawdata[i:endpos]) + return endpos + if end.endswith('/>'): + # XHTML-style empty tag: <span attr="value" /> + self.handle_startendtag(tag, attrs) + else: + self.handle_starttag(tag, attrs) + if tag in self.CDATA_CONTENT_ELEMENTS: + self.set_cdata_mode(tag) + return endpos + + def set_cdata_mode(self, elem): + self.cdata_elem = elem.lower() + self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I) + + BeautifulSoupHTMLParser.parse_starttag = parse_starttag + BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode + + CONSTRUCTOR_TAKES_STRICT = True diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/builder/_lxml.py b/import-layers/yocto-poky/bitbake/lib/bs4/builder/_lxml.py new file mode 100644 index 000000000..fa5d49875 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/builder/_lxml.py @@ -0,0 +1,233 @@ +__all__ = [ + 'LXMLTreeBuilderForXML', + 'LXMLTreeBuilder', + ] + +from io import BytesIO +from StringIO import StringIO +import collections +from lxml import etree +from bs4.element import Comment, Doctype, NamespacedAttribute +from bs4.builder import ( + FAST, + HTML, + HTMLTreeBuilder, + PERMISSIVE, + ParserRejectedMarkup, + TreeBuilder, + XML) +from bs4.dammit import EncodingDetector + +LXML = 'lxml' + +class LXMLTreeBuilderForXML(TreeBuilder): + DEFAULT_PARSER_CLASS = etree.XMLParser + + is_xml = True + + # Well, it's permissive by XML parser standards. + features = [LXML, XML, FAST, PERMISSIVE] + + CHUNK_SIZE = 512 + + # This namespace mapping is specified in the XML Namespace + # standard. + DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"} + + def default_parser(self, encoding): + # This can either return a parser object or a class, which + # will be instantiated with default arguments. + if self._default_parser is not None: + return self._default_parser + return etree.XMLParser( + target=self, strip_cdata=False, recover=True, encoding=encoding) + + def parser_for(self, encoding): + # Use the default parser. + parser = self.default_parser(encoding) + + if isinstance(parser, collections.Callable): + # Instantiate the parser with default arguments + parser = parser(target=self, strip_cdata=False, encoding=encoding) + return parser + + def __init__(self, parser=None, empty_element_tags=None): + # TODO: Issue a warning if parser is present but not a + # callable, since that means there's no way to create new + # parsers for different encodings. + self._default_parser = parser + if empty_element_tags is not None: + self.empty_element_tags = set(empty_element_tags) + self.soup = None + self.nsmaps = [self.DEFAULT_NSMAPS] + + def _getNsTag(self, tag): + # Split the namespace URL out of a fully-qualified lxml tag + # name. Copied from lxml's src/lxml/sax.py. + if tag[0] == '{': + return tuple(tag[1:].split('}', 1)) + else: + return (None, tag) + + def prepare_markup(self, markup, user_specified_encoding=None, + document_declared_encoding=None): + """ + :yield: A series of 4-tuples. + (markup, encoding, declared encoding, + has undergone character replacement) + + Each 4-tuple represents a strategy for parsing the document. + """ + if isinstance(markup, unicode): + # We were given Unicode. Maybe lxml can parse Unicode on + # this system? + yield markup, None, document_declared_encoding, False + + if isinstance(markup, unicode): + # No, apparently not. Convert the Unicode to UTF-8 and + # tell lxml to parse it as UTF-8. + yield (markup.encode("utf8"), "utf8", + document_declared_encoding, False) + + # Instead of using UnicodeDammit to convert the bytestring to + # Unicode using different encodings, use EncodingDetector to + # iterate over the encodings, and tell lxml to try to parse + # the document as each one in turn. + is_html = not self.is_xml + try_encodings = [user_specified_encoding, document_declared_encoding] + detector = EncodingDetector(markup, try_encodings, is_html) + for encoding in detector.encodings: + yield (detector.markup, encoding, document_declared_encoding, False) + + def feed(self, markup): + if isinstance(markup, bytes): + markup = BytesIO(markup) + elif isinstance(markup, unicode): + markup = StringIO(markup) + + # Call feed() at least once, even if the markup is empty, + # or the parser won't be initialized. + data = markup.read(self.CHUNK_SIZE) + try: + self.parser = self.parser_for(self.soup.original_encoding) + self.parser.feed(data) + while len(data) != 0: + # Now call feed() on the rest of the data, chunk by chunk. + data = markup.read(self.CHUNK_SIZE) + if len(data) != 0: + self.parser.feed(data) + self.parser.close() + except (UnicodeDecodeError, LookupError, etree.ParserError), e: + raise ParserRejectedMarkup(str(e)) + + def close(self): + self.nsmaps = [self.DEFAULT_NSMAPS] + + def start(self, name, attrs, nsmap={}): + # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy. + attrs = dict(attrs) + nsprefix = None + # Invert each namespace map as it comes in. + if len(self.nsmaps) > 1: + # There are no new namespaces for this tag, but + # non-default namespaces are in play, so we need a + # separate tag stack to know when they end. + self.nsmaps.append(None) + elif len(nsmap) > 0: + # A new namespace mapping has come into play. + inverted_nsmap = dict((value, key) for key, value in nsmap.items()) + self.nsmaps.append(inverted_nsmap) + # Also treat the namespace mapping as a set of attributes on the + # tag, so we can recreate it later. + attrs = attrs.copy() + for prefix, namespace in nsmap.items(): + attribute = NamespacedAttribute( + "xmlns", prefix, "http://www.w3.org/2000/xmlns/") + attrs[attribute] = namespace + + # Namespaces are in play. Find any attributes that came in + # from lxml with namespaces attached to their names, and + # turn then into NamespacedAttribute objects. + new_attrs = {} + for attr, value in attrs.items(): + namespace, attr = self._getNsTag(attr) + if namespace is None: + new_attrs[attr] = value + else: + nsprefix = self._prefix_for_namespace(namespace) + attr = NamespacedAttribute(nsprefix, attr, namespace) + new_attrs[attr] = value + attrs = new_attrs + + namespace, name = self._getNsTag(name) + nsprefix = self._prefix_for_namespace(namespace) + self.soup.handle_starttag(name, namespace, nsprefix, attrs) + + def _prefix_for_namespace(self, namespace): + """Find the currently active prefix for the given namespace.""" + if namespace is None: + return None + for inverted_nsmap in reversed(self.nsmaps): + if inverted_nsmap is not None and namespace in inverted_nsmap: + return inverted_nsmap[namespace] + return None + + def end(self, name): + self.soup.endData() + completed_tag = self.soup.tagStack[-1] + namespace, name = self._getNsTag(name) + nsprefix = None + if namespace is not None: + for inverted_nsmap in reversed(self.nsmaps): + if inverted_nsmap is not None and namespace in inverted_nsmap: + nsprefix = inverted_nsmap[namespace] + break + self.soup.handle_endtag(name, nsprefix) + if len(self.nsmaps) > 1: + # This tag, or one of its parents, introduced a namespace + # mapping, so pop it off the stack. + self.nsmaps.pop() + + def pi(self, target, data): + pass + + def data(self, content): + self.soup.handle_data(content) + + def doctype(self, name, pubid, system): + self.soup.endData() + doctype = Doctype.for_name_and_ids(name, pubid, system) + self.soup.object_was_parsed(doctype) + + def comment(self, content): + "Handle comments as Comment objects." + self.soup.endData() + self.soup.handle_data(content) + self.soup.endData(Comment) + + def test_fragment_to_document(self, fragment): + """See `TreeBuilder`.""" + return u'<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment + + +class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML): + + features = [LXML, HTML, FAST, PERMISSIVE] + is_xml = False + + def default_parser(self, encoding): + return etree.HTMLParser + + def feed(self, markup): + encoding = self.soup.original_encoding + try: + self.parser = self.parser_for(encoding) + self.parser.feed(markup) + self.parser.close() + except (UnicodeDecodeError, LookupError, etree.ParserError), e: + raise ParserRejectedMarkup(str(e)) + + + def test_fragment_to_document(self, fragment): + """See `TreeBuilder`.""" + return u'<html><body>%s</body></html>' % fragment diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/dammit.py b/import-layers/yocto-poky/bitbake/lib/bs4/dammit.py new file mode 100644 index 000000000..59640b7ce --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/dammit.py @@ -0,0 +1,829 @@ +# -*- coding: utf-8 -*- +"""Beautiful Soup bonus library: Unicode, Dammit + +This library converts a bytestream to Unicode through any means +necessary. It is heavily based on code from Mark Pilgrim's Universal +Feed Parser. It works best on XML and XML, but it does not rewrite the +XML or HTML to reflect a new encoding; that's the tree builder's job. +""" + +import codecs +from htmlentitydefs import codepoint2name +import re +import logging +import string + +# Import a library to autodetect character encodings. +chardet_type = None +try: + # First try the fast C implementation. + # PyPI package: cchardet + import cchardet + def chardet_dammit(s): + return cchardet.detect(s)['encoding'] +except ImportError: + try: + # Fall back to the pure Python implementation + # Debian package: python-chardet + # PyPI package: chardet + import chardet + def chardet_dammit(s): + return chardet.detect(s)['encoding'] + #import chardet.constants + #chardet.constants._debug = 1 + except ImportError: + # No chardet available. + def chardet_dammit(s): + return None + +# Available from http://cjkpython.i18n.org/. +try: + import iconv_codec +except ImportError: + pass + +xml_encoding_re = re.compile( + '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I) +html_meta_re = re.compile( + '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I) + +class EntitySubstitution(object): + + """Substitute XML or HTML entities for the corresponding characters.""" + + def _populate_class_variables(): + lookup = {} + reverse_lookup = {} + characters_for_re = [] + for codepoint, name in list(codepoint2name.items()): + character = unichr(codepoint) + if codepoint != 34: + # There's no point in turning the quotation mark into + # ", unless it happens within an attribute value, which + # is handled elsewhere. + characters_for_re.append(character) + lookup[character] = name + # But we do want to turn " into the quotation mark. + reverse_lookup[name] = character + re_definition = "[%s]" % "".join(characters_for_re) + return lookup, reverse_lookup, re.compile(re_definition) + (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER, + CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables() + + CHARACTER_TO_XML_ENTITY = { + "'": "apos", + '"': "quot", + "&": "amp", + "<": "lt", + ">": "gt", + } + + BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|" + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)" + ")") + + AMPERSAND_OR_BRACKET = re.compile("([<>&])") + + @classmethod + def _substitute_html_entity(cls, matchobj): + entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0)) + return "&%s;" % entity + + @classmethod + def _substitute_xml_entity(cls, matchobj): + """Used with a regular expression to substitute the + appropriate XML entity for an XML special character.""" + entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)] + return "&%s;" % entity + + @classmethod + def quoted_attribute_value(self, value): + """Make a value into a quoted XML attribute, possibly escaping it. + + Most strings will be quoted using double quotes. + + Bob's Bar -> "Bob's Bar" + + If a string contains double quotes, it will be quoted using + single quotes. + + Welcome to "my bar" -> 'Welcome to "my bar"' + + If a string contains both single and double quotes, the + double quotes will be escaped, and the string will be quoted + using double quotes. + + Welcome to "Bob's Bar" -> "Welcome to "Bob's bar" + """ + quote_with = '"' + if '"' in value: + if "'" in value: + # The string contains both single and double + # quotes. Turn the double quotes into + # entities. We quote the double quotes rather than + # the single quotes because the entity name is + # """ whether this is HTML or XML. If we + # quoted the single quotes, we'd have to decide + # between ' and &squot;. + replace_with = """ + value = value.replace('"', replace_with) + else: + # There are double quotes but no single quotes. + # We can use single quotes to quote the attribute. + quote_with = "'" + return quote_with + value + quote_with + + @classmethod + def substitute_xml(cls, value, make_quoted_attribute=False): + """Substitute XML entities for special XML characters. + + :param value: A string to be substituted. The less-than sign + will become <, the greater-than sign will become >, + and any ampersands will become &. If you want ampersands + that appear to be part of an entity definition to be left + alone, use substitute_xml_containing_entities() instead. + + :param make_quoted_attribute: If True, then the string will be + quoted, as befits an attribute value. + """ + # Escape angle brackets and ampersands. + value = cls.AMPERSAND_OR_BRACKET.sub( + cls._substitute_xml_entity, value) + + if make_quoted_attribute: + value = cls.quoted_attribute_value(value) + return value + + @classmethod + def substitute_xml_containing_entities( + cls, value, make_quoted_attribute=False): + """Substitute XML entities for special XML characters. + + :param value: A string to be substituted. The less-than sign will + become <, the greater-than sign will become >, and any + ampersands that are not part of an entity defition will + become &. + + :param make_quoted_attribute: If True, then the string will be + quoted, as befits an attribute value. + """ + # Escape angle brackets, and ampersands that aren't part of + # entities. + value = cls.BARE_AMPERSAND_OR_BRACKET.sub( + cls._substitute_xml_entity, value) + + if make_quoted_attribute: + value = cls.quoted_attribute_value(value) + return value + + @classmethod + def substitute_html(cls, s): + """Replace certain Unicode characters with named HTML entities. + + This differs from data.encode(encoding, 'xmlcharrefreplace') + in that the goal is to make the result more readable (to those + with ASCII displays) rather than to recover from + errors. There's absolutely nothing wrong with a UTF-8 string + containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that + character with "é" will make it more readable to some + people. + """ + return cls.CHARACTER_TO_HTML_ENTITY_RE.sub( + cls._substitute_html_entity, s) + + +class EncodingDetector: + """Suggests a number of possible encodings for a bytestring. + + Order of precedence: + + 1. Encodings you specifically tell EncodingDetector to try first + (the override_encodings argument to the constructor). + + 2. An encoding declared within the bytestring itself, either in an + XML declaration (if the bytestring is to be interpreted as an XML + document), or in a <meta> tag (if the bytestring is to be + interpreted as an HTML document.) + + 3. An encoding detected through textual analysis by chardet, + cchardet, or a similar external library. + + 4. UTF-8. + + 5. Windows-1252. + """ + def __init__(self, markup, override_encodings=None, is_html=False): + self.override_encodings = override_encodings or [] + self.chardet_encoding = None + self.is_html = is_html + self.declared_encoding = None + + # First order of business: strip a byte-order mark. + self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup) + + def _usable(self, encoding, tried): + if encoding is not None: + encoding = encoding.lower() + if encoding not in tried: + tried.add(encoding) + return True + return False + + @property + def encodings(self): + """Yield a number of encodings that might work for this markup.""" + tried = set() + for e in self.override_encodings: + if self._usable(e, tried): + yield e + + # Did the document originally start with a byte-order mark + # that indicated its encoding? + if self._usable(self.sniffed_encoding, tried): + yield self.sniffed_encoding + + # Look within the document for an XML or HTML encoding + # declaration. + if self.declared_encoding is None: + self.declared_encoding = self.find_declared_encoding( + self.markup, self.is_html) + if self._usable(self.declared_encoding, tried): + yield self.declared_encoding + + # Use third-party character set detection to guess at the + # encoding. + if self.chardet_encoding is None: + self.chardet_encoding = chardet_dammit(self.markup) + if self._usable(self.chardet_encoding, tried): + yield self.chardet_encoding + + # As a last-ditch effort, try utf-8 and windows-1252. + for e in ('utf-8', 'windows-1252'): + if self._usable(e, tried): + yield e + + @classmethod + def strip_byte_order_mark(cls, data): + """If a byte-order mark is present, strip it and return the encoding it implies.""" + encoding = None + if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \ + and (data[2:4] != '\x00\x00'): + encoding = 'utf-16be' + data = data[2:] + elif (len(data) >= 4) and (data[:2] == b'\xff\xfe') \ + and (data[2:4] != '\x00\x00'): + encoding = 'utf-16le' + data = data[2:] + elif data[:3] == b'\xef\xbb\xbf': + encoding = 'utf-8' + data = data[3:] + elif data[:4] == b'\x00\x00\xfe\xff': + encoding = 'utf-32be' + data = data[4:] + elif data[:4] == b'\xff\xfe\x00\x00': + encoding = 'utf-32le' + data = data[4:] + return data, encoding + + @classmethod + def find_declared_encoding(cls, markup, is_html=False, search_entire_document=False): + """Given a document, tries to find its declared encoding. + + An XML encoding is declared at the beginning of the document. + + An HTML encoding is declared in a <meta> tag, hopefully near the + beginning of the document. + """ + if search_entire_document: + xml_endpos = html_endpos = len(markup) + else: + xml_endpos = 1024 + html_endpos = max(2048, int(len(markup) * 0.05)) + + declared_encoding = None + declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos) + if not declared_encoding_match and is_html: + declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos) + if declared_encoding_match is not None: + declared_encoding = declared_encoding_match.groups()[0].decode( + 'ascii') + if declared_encoding: + return declared_encoding.lower() + return None + +class UnicodeDammit: + """A class for detecting the encoding of a *ML document and + converting it to a Unicode string. If the source encoding is + windows-1252, can replace MS smart quotes with their HTML or XML + equivalents.""" + + # This dictionary maps commonly seen values for "charset" in HTML + # meta tags to the corresponding Python codec names. It only covers + # values that aren't in Python's aliases and can't be determined + # by the heuristics in find_codec. + CHARSET_ALIASES = {"macintosh": "mac-roman", + "x-sjis": "shift-jis"} + + ENCODINGS_WITH_SMART_QUOTES = [ + "windows-1252", + "iso-8859-1", + "iso-8859-2", + ] + + def __init__(self, markup, override_encodings=[], + smart_quotes_to=None, is_html=False): + self.smart_quotes_to = smart_quotes_to + self.tried_encodings = [] + self.contains_replacement_characters = False + self.is_html = is_html + + self.detector = EncodingDetector(markup, override_encodings, is_html) + + # Short-circuit if the data is in Unicode to begin with. + if isinstance(markup, unicode) or markup == '': + self.markup = markup + self.unicode_markup = unicode(markup) + self.original_encoding = None + return + + # The encoding detector may have stripped a byte-order mark. + # Use the stripped markup from this point on. + self.markup = self.detector.markup + + u = None + for encoding in self.detector.encodings: + markup = self.detector.markup + u = self._convert_from(encoding) + if u is not None: + break + + if not u: + # None of the encodings worked. As an absolute last resort, + # try them again with character replacement. + + for encoding in self.detector.encodings: + if encoding != "ascii": + u = self._convert_from(encoding, "replace") + if u is not None: + logging.warning( + "Some characters could not be decoded, and were " + "replaced with REPLACEMENT CHARACTER.") + self.contains_replacement_characters = True + break + + # If none of that worked, we could at this point force it to + # ASCII, but that would destroy so much data that I think + # giving up is better. + self.unicode_markup = u + if not u: + self.original_encoding = None + + def _sub_ms_char(self, match): + """Changes a MS smart quote character to an XML or HTML + entity, or an ASCII character.""" + orig = match.group(1) + if self.smart_quotes_to == 'ascii': + sub = self.MS_CHARS_TO_ASCII.get(orig).encode() + else: + sub = self.MS_CHARS.get(orig) + if type(sub) == tuple: + if self.smart_quotes_to == 'xml': + sub = '&#x'.encode() + sub[1].encode() + ';'.encode() + else: + sub = '&'.encode() + sub[0].encode() + ';'.encode() + else: + sub = sub.encode() + return sub + + def _convert_from(self, proposed, errors="strict"): + proposed = self.find_codec(proposed) + if not proposed or (proposed, errors) in self.tried_encodings: + return None + self.tried_encodings.append((proposed, errors)) + markup = self.markup + # Convert smart quotes to HTML if coming from an encoding + # that might have them. + if (self.smart_quotes_to is not None + and proposed in self.ENCODINGS_WITH_SMART_QUOTES): + smart_quotes_re = b"([\x80-\x9f])" + smart_quotes_compiled = re.compile(smart_quotes_re) + markup = smart_quotes_compiled.sub(self._sub_ms_char, markup) + + try: + #print "Trying to convert document to %s (errors=%s)" % ( + # proposed, errors) + u = self._to_unicode(markup, proposed, errors) + self.markup = u + self.original_encoding = proposed + except Exception as e: + #print "That didn't work!" + #print e + return None + #print "Correct encoding: %s" % proposed + return self.markup + + def _to_unicode(self, data, encoding, errors="strict"): + '''Given a string and its encoding, decodes the string into Unicode. + %encoding is a string recognized by encodings.aliases''' + return unicode(data, encoding, errors) + + @property + def declared_html_encoding(self): + if not self.is_html: + return None + return self.detector.declared_encoding + + def find_codec(self, charset): + value = (self._codec(self.CHARSET_ALIASES.get(charset, charset)) + or (charset and self._codec(charset.replace("-", ""))) + or (charset and self._codec(charset.replace("-", "_"))) + or (charset and charset.lower()) + or charset + ) + if value: + return value.lower() + return None + + def _codec(self, charset): + if not charset: + return charset + codec = None + try: + codecs.lookup(charset) + codec = charset + except (LookupError, ValueError): + pass + return codec + + + # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities. + MS_CHARS = {b'\x80': ('euro', '20AC'), + b'\x81': ' ', + b'\x82': ('sbquo', '201A'), + b'\x83': ('fnof', '192'), + b'\x84': ('bdquo', '201E'), + b'\x85': ('hellip', '2026'), + b'\x86': ('dagger', '2020'), + b'\x87': ('Dagger', '2021'), + b'\x88': ('circ', '2C6'), + b'\x89': ('permil', '2030'), + b'\x8A': ('Scaron', '160'), + b'\x8B': ('lsaquo', '2039'), + b'\x8C': ('OElig', '152'), + b'\x8D': '?', + b'\x8E': ('#x17D', '17D'), + b'\x8F': '?', + b'\x90': '?', + b'\x91': ('lsquo', '2018'), + b'\x92': ('rsquo', '2019'), + b'\x93': ('ldquo', '201C'), + b'\x94': ('rdquo', '201D'), + b'\x95': ('bull', '2022'), + b'\x96': ('ndash', '2013'), + b'\x97': ('mdash', '2014'), + b'\x98': ('tilde', '2DC'), + b'\x99': ('trade', '2122'), + b'\x9a': ('scaron', '161'), + b'\x9b': ('rsaquo', '203A'), + b'\x9c': ('oelig', '153'), + b'\x9d': '?', + b'\x9e': ('#x17E', '17E'), + b'\x9f': ('Yuml', ''),} + + # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains + # horrors like stripping diacritical marks to turn á into a, but also + # contains non-horrors like turning “ into ". + MS_CHARS_TO_ASCII = { + b'\x80' : 'EUR', + b'\x81' : ' ', + b'\x82' : ',', + b'\x83' : 'f', + b'\x84' : ',,', + b'\x85' : '...', + b'\x86' : '+', + b'\x87' : '++', + b'\x88' : '^', + b'\x89' : '%', + b'\x8a' : 'S', + b'\x8b' : '<', + b'\x8c' : 'OE', + b'\x8d' : '?', + b'\x8e' : 'Z', + b'\x8f' : '?', + b'\x90' : '?', + b'\x91' : "'", + b'\x92' : "'", + b'\x93' : '"', + b'\x94' : '"', + b'\x95' : '*', + b'\x96' : '-', + b'\x97' : '--', + b'\x98' : '~', + b'\x99' : '(TM)', + b'\x9a' : 's', + b'\x9b' : '>', + b'\x9c' : 'oe', + b'\x9d' : '?', + b'\x9e' : 'z', + b'\x9f' : 'Y', + b'\xa0' : ' ', + b'\xa1' : '!', + b'\xa2' : 'c', + b'\xa3' : 'GBP', + b'\xa4' : '$', #This approximation is especially parochial--this is the + #generic currency symbol. + b'\xa5' : 'YEN', + b'\xa6' : '|', + b'\xa7' : 'S', + b'\xa8' : '..', + b'\xa9' : '', + b'\xaa' : '(th)', + b'\xab' : '<<', + b'\xac' : '!', + b'\xad' : ' ', + b'\xae' : '(R)', + b'\xaf' : '-', + b'\xb0' : 'o', + b'\xb1' : '+-', + b'\xb2' : '2', + b'\xb3' : '3', + b'\xb4' : ("'", 'acute'), + b'\xb5' : 'u', + b'\xb6' : 'P', + b'\xb7' : '*', + b'\xb8' : ',', + b'\xb9' : '1', + b'\xba' : '(th)', + b'\xbb' : '>>', + b'\xbc' : '1/4', + b'\xbd' : '1/2', + b'\xbe' : '3/4', + b'\xbf' : '?', + b'\xc0' : 'A', + b'\xc1' : 'A', + b'\xc2' : 'A', + b'\xc3' : 'A', + b'\xc4' : 'A', + b'\xc5' : 'A', + b'\xc6' : 'AE', + b'\xc7' : 'C', + b'\xc8' : 'E', + b'\xc9' : 'E', + b'\xca' : 'E', + b'\xcb' : 'E', + b'\xcc' : 'I', + b'\xcd' : 'I', + b'\xce' : 'I', + b'\xcf' : 'I', + b'\xd0' : 'D', + b'\xd1' : 'N', + b'\xd2' : 'O', + b'\xd3' : 'O', + b'\xd4' : 'O', + b'\xd5' : 'O', + b'\xd6' : 'O', + b'\xd7' : '*', + b'\xd8' : 'O', + b'\xd9' : 'U', + b'\xda' : 'U', + b'\xdb' : 'U', + b'\xdc' : 'U', + b'\xdd' : 'Y', + b'\xde' : 'b', + b'\xdf' : 'B', + b'\xe0' : 'a', + b'\xe1' : 'a', + b'\xe2' : 'a', + b'\xe3' : 'a', + b'\xe4' : 'a', + b'\xe5' : 'a', + b'\xe6' : 'ae', + b'\xe7' : 'c', + b'\xe8' : 'e', + b'\xe9' : 'e', + b'\xea' : 'e', + b'\xeb' : 'e', + b'\xec' : 'i', + b'\xed' : 'i', + b'\xee' : 'i', + b'\xef' : 'i', + b'\xf0' : 'o', + b'\xf1' : 'n', + b'\xf2' : 'o', + b'\xf3' : 'o', + b'\xf4' : 'o', + b'\xf5' : 'o', + b'\xf6' : 'o', + b'\xf7' : '/', + b'\xf8' : 'o', + b'\xf9' : 'u', + b'\xfa' : 'u', + b'\xfb' : 'u', + b'\xfc' : 'u', + b'\xfd' : 'y', + b'\xfe' : 'b', + b'\xff' : 'y', + } + + # A map used when removing rogue Windows-1252/ISO-8859-1 + # characters in otherwise UTF-8 documents. + # + # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in + # Windows-1252. + WINDOWS_1252_TO_UTF8 = { + 0x80 : b'\xe2\x82\xac', # € + 0x82 : b'\xe2\x80\x9a', # ‚ + 0x83 : b'\xc6\x92', # Æ’ + 0x84 : b'\xe2\x80\x9e', # „ + 0x85 : b'\xe2\x80\xa6', # … + 0x86 : b'\xe2\x80\xa0', # † + 0x87 : b'\xe2\x80\xa1', # ‡ + 0x88 : b'\xcb\x86', # ˆ + 0x89 : b'\xe2\x80\xb0', # ‰ + 0x8a : b'\xc5\xa0', # Å  + 0x8b : b'\xe2\x80\xb9', # ‹ + 0x8c : b'\xc5\x92', # Å’ + 0x8e : b'\xc5\xbd', # Ž + 0x91 : b'\xe2\x80\x98', # ‘ + 0x92 : b'\xe2\x80\x99', # ’ + 0x93 : b'\xe2\x80\x9c', # “ + 0x94 : b'\xe2\x80\x9d', # †+ 0x95 : b'\xe2\x80\xa2', # • + 0x96 : b'\xe2\x80\x93', # – + 0x97 : b'\xe2\x80\x94', # — + 0x98 : b'\xcb\x9c', # Ëœ + 0x99 : b'\xe2\x84\xa2', # â„¢ + 0x9a : b'\xc5\xa1', # Å¡ + 0x9b : b'\xe2\x80\xba', # › + 0x9c : b'\xc5\x93', # Å“ + 0x9e : b'\xc5\xbe', # ž + 0x9f : b'\xc5\xb8', # Ÿ + 0xa0 : b'\xc2\xa0', #   + 0xa1 : b'\xc2\xa1', # ¡ + 0xa2 : b'\xc2\xa2', # ¢ + 0xa3 : b'\xc2\xa3', # £ + 0xa4 : b'\xc2\xa4', # ¤ + 0xa5 : b'\xc2\xa5', # Â¥ + 0xa6 : b'\xc2\xa6', # ¦ + 0xa7 : b'\xc2\xa7', # § + 0xa8 : b'\xc2\xa8', # ¨ + 0xa9 : b'\xc2\xa9', # © + 0xaa : b'\xc2\xaa', # ª + 0xab : b'\xc2\xab', # « + 0xac : b'\xc2\xac', # ¬ + 0xad : b'\xc2\xad', # ­ + 0xae : b'\xc2\xae', # ® + 0xaf : b'\xc2\xaf', # ¯ + 0xb0 : b'\xc2\xb0', # ° + 0xb1 : b'\xc2\xb1', # ± + 0xb2 : b'\xc2\xb2', # ² + 0xb3 : b'\xc2\xb3', # ³ + 0xb4 : b'\xc2\xb4', # ´ + 0xb5 : b'\xc2\xb5', # µ + 0xb6 : b'\xc2\xb6', # ¶ + 0xb7 : b'\xc2\xb7', # · + 0xb8 : b'\xc2\xb8', # ¸ + 0xb9 : b'\xc2\xb9', # ¹ + 0xba : b'\xc2\xba', # º + 0xbb : b'\xc2\xbb', # » + 0xbc : b'\xc2\xbc', # ¼ + 0xbd : b'\xc2\xbd', # ½ + 0xbe : b'\xc2\xbe', # ¾ + 0xbf : b'\xc2\xbf', # ¿ + 0xc0 : b'\xc3\x80', # À + 0xc1 : b'\xc3\x81', # à + 0xc2 : b'\xc3\x82', #  + 0xc3 : b'\xc3\x83', # à + 0xc4 : b'\xc3\x84', # Ä + 0xc5 : b'\xc3\x85', # Ã… + 0xc6 : b'\xc3\x86', # Æ + 0xc7 : b'\xc3\x87', # Ç + 0xc8 : b'\xc3\x88', # È + 0xc9 : b'\xc3\x89', # É + 0xca : b'\xc3\x8a', # Ê + 0xcb : b'\xc3\x8b', # Ë + 0xcc : b'\xc3\x8c', # ÃŒ + 0xcd : b'\xc3\x8d', # à + 0xce : b'\xc3\x8e', # ÃŽ + 0xcf : b'\xc3\x8f', # à + 0xd0 : b'\xc3\x90', # à + 0xd1 : b'\xc3\x91', # Ñ + 0xd2 : b'\xc3\x92', # Ã’ + 0xd3 : b'\xc3\x93', # Ó + 0xd4 : b'\xc3\x94', # Ô + 0xd5 : b'\xc3\x95', # Õ + 0xd6 : b'\xc3\x96', # Ö + 0xd7 : b'\xc3\x97', # × + 0xd8 : b'\xc3\x98', # Ø + 0xd9 : b'\xc3\x99', # Ù + 0xda : b'\xc3\x9a', # Ú + 0xdb : b'\xc3\x9b', # Û + 0xdc : b'\xc3\x9c', # Ãœ + 0xdd : b'\xc3\x9d', # à + 0xde : b'\xc3\x9e', # Þ + 0xdf : b'\xc3\x9f', # ß + 0xe0 : b'\xc3\xa0', # à + 0xe1 : b'\xa1', # á + 0xe2 : b'\xc3\xa2', # â + 0xe3 : b'\xc3\xa3', # ã + 0xe4 : b'\xc3\xa4', # ä + 0xe5 : b'\xc3\xa5', # Ã¥ + 0xe6 : b'\xc3\xa6', # æ + 0xe7 : b'\xc3\xa7', # ç + 0xe8 : b'\xc3\xa8', # è + 0xe9 : b'\xc3\xa9', # é + 0xea : b'\xc3\xaa', # ê + 0xeb : b'\xc3\xab', # ë + 0xec : b'\xc3\xac', # ì + 0xed : b'\xc3\xad', # í + 0xee : b'\xc3\xae', # î + 0xef : b'\xc3\xaf', # ï + 0xf0 : b'\xc3\xb0', # ð + 0xf1 : b'\xc3\xb1', # ñ + 0xf2 : b'\xc3\xb2', # ò + 0xf3 : b'\xc3\xb3', # ó + 0xf4 : b'\xc3\xb4', # ô + 0xf5 : b'\xc3\xb5', # õ + 0xf6 : b'\xc3\xb6', # ö + 0xf7 : b'\xc3\xb7', # ÷ + 0xf8 : b'\xc3\xb8', # ø + 0xf9 : b'\xc3\xb9', # ù + 0xfa : b'\xc3\xba', # ú + 0xfb : b'\xc3\xbb', # û + 0xfc : b'\xc3\xbc', # ü + 0xfd : b'\xc3\xbd', # ý + 0xfe : b'\xc3\xbe', # þ + } + + MULTIBYTE_MARKERS_AND_SIZES = [ + (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF + (0xe0, 0xef, 3), # 3-byte characters start with E0-EF + (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4 + ] + + FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0] + LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1] + + @classmethod + def detwingle(cls, in_bytes, main_encoding="utf8", + embedded_encoding="windows-1252"): + """Fix characters from one encoding embedded in some other encoding. + + Currently the only situation supported is Windows-1252 (or its + subset ISO-8859-1), embedded in UTF-8. + + The input must be a bytestring. If you've already converted + the document to Unicode, you're too late. + + The output is a bytestring in which `embedded_encoding` + characters have been converted to their `main_encoding` + equivalents. + """ + if embedded_encoding.replace('_', '-').lower() not in ( + 'windows-1252', 'windows_1252'): + raise NotImplementedError( + "Windows-1252 and ISO-8859-1 are the only currently supported " + "embedded encodings.") + + if main_encoding.lower() not in ('utf8', 'utf-8'): + raise NotImplementedError( + "UTF-8 is the only currently supported main encoding.") + + byte_chunks = [] + + chunk_start = 0 + pos = 0 + while pos < len(in_bytes): + byte = in_bytes[pos] + if not isinstance(byte, int): + # Python 2.x + byte = ord(byte) + if (byte >= cls.FIRST_MULTIBYTE_MARKER + and byte <= cls.LAST_MULTIBYTE_MARKER): + # This is the start of a UTF-8 multibyte character. Skip + # to the end. + for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES: + if byte >= start and byte <= end: + pos += size + break + elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8: + # We found a Windows-1252 character! + # Save the string up to this point as a chunk. + byte_chunks.append(in_bytes[chunk_start:pos]) + + # Now translate the Windows-1252 character into UTF-8 + # and add it as another, one-byte chunk. + byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte]) + pos += 1 + chunk_start = pos + else: + # Go on to the next character. + pos += 1 + if chunk_start == 0: + # The string is unchanged. + return in_bytes + else: + # Store the final chunk. + byte_chunks.append(in_bytes[chunk_start:]) + return b''.join(byte_chunks) + diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/diagnose.py b/import-layers/yocto-poky/bitbake/lib/bs4/diagnose.py new file mode 100644 index 000000000..4d0b00afa --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/diagnose.py @@ -0,0 +1,204 @@ +"""Diagnostic functions, mainly for use when doing tech support.""" +import cProfile +from StringIO import StringIO +from HTMLParser import HTMLParser +import bs4 +from bs4 import BeautifulSoup, __version__ +from bs4.builder import builder_registry + +import os +import pstats +import random +import tempfile +import time +import traceback +import sys +import cProfile + +def diagnose(data): + """Diagnostic suite for isolating common problems.""" + print "Diagnostic running on Beautiful Soup %s" % __version__ + print "Python version %s" % sys.version + + basic_parsers = ["html.parser", "html5lib", "lxml"] + for name in basic_parsers: + for builder in builder_registry.builders: + if name in builder.features: + break + else: + basic_parsers.remove(name) + print ( + "I noticed that %s is not installed. Installing it may help." % + name) + + if 'lxml' in basic_parsers: + basic_parsers.append(["lxml", "xml"]) + from lxml import etree + print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION)) + + if 'html5lib' in basic_parsers: + import html5lib + print "Found html5lib version %s" % html5lib.__version__ + + if hasattr(data, 'read'): + data = data.read() + elif os.path.exists(data): + print '"%s" looks like a filename. Reading data from the file.' % data + data = open(data).read() + elif data.startswith("http:") or data.startswith("https:"): + print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data + print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup." + return + print + + for parser in basic_parsers: + print "Trying to parse your markup with %s" % parser + success = False + try: + soup = BeautifulSoup(data, parser) + success = True + except Exception, e: + print "%s could not parse the markup." % parser + traceback.print_exc() + if success: + print "Here's what %s did with the markup:" % parser + print soup.prettify() + + print "-" * 80 + +def lxml_trace(data, html=True, **kwargs): + """Print out the lxml events that occur during parsing. + + This lets you see how lxml parses a document when no Beautiful + Soup code is running. + """ + from lxml import etree + for event, element in etree.iterparse(StringIO(data), html=html, **kwargs): + print("%s, %4s, %s" % (event, element.tag, element.text)) + +class AnnouncingParser(HTMLParser): + """Announces HTMLParser parse events, without doing anything else.""" + + def _p(self, s): + print(s) + + def handle_starttag(self, name, attrs): + self._p("%s START" % name) + + def handle_endtag(self, name): + self._p("%s END" % name) + + def handle_data(self, data): + self._p("%s DATA" % data) + + def handle_charref(self, name): + self._p("%s CHARREF" % name) + + def handle_entityref(self, name): + self._p("%s ENTITYREF" % name) + + def handle_comment(self, data): + self._p("%s COMMENT" % data) + + def handle_decl(self, data): + self._p("%s DECL" % data) + + def unknown_decl(self, data): + self._p("%s UNKNOWN-DECL" % data) + + def handle_pi(self, data): + self._p("%s PI" % data) + +def htmlparser_trace(data): + """Print out the HTMLParser events that occur during parsing. + + This lets you see how HTMLParser parses a document when no + Beautiful Soup code is running. + """ + parser = AnnouncingParser() + parser.feed(data) + +_vowels = "aeiou" +_consonants = "bcdfghjklmnpqrstvwxyz" + +def rword(length=5): + "Generate a random word-like string." + s = '' + for i in range(length): + if i % 2 == 0: + t = _consonants + else: + t = _vowels + s += random.choice(t) + return s + +def rsentence(length=4): + "Generate a random sentence-like string." + return " ".join(rword(random.randint(4,9)) for i in range(length)) + +def rdoc(num_elements=1000): + """Randomly generate an invalid HTML document.""" + tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table'] + elements = [] + for i in range(num_elements): + choice = random.randint(0,3) + if choice == 0: + # New tag. + tag_name = random.choice(tag_names) + elements.append("<%s>" % tag_name) + elif choice == 1: + elements.append(rsentence(random.randint(1,4))) + elif choice == 2: + # Close a tag. + tag_name = random.choice(tag_names) + elements.append("</%s>" % tag_name) + return "<html>" + "\n".join(elements) + "</html>" + +def benchmark_parsers(num_elements=100000): + """Very basic head-to-head performance benchmark.""" + print "Comparative parser benchmark on Beautiful Soup %s" % __version__ + data = rdoc(num_elements) + print "Generated a large invalid HTML document (%d bytes)." % len(data) + + for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]: + success = False + try: + a = time.time() + soup = BeautifulSoup(data, parser) + b = time.time() + success = True + except Exception, e: + print "%s could not parse the markup." % parser + traceback.print_exc() + if success: + print "BS4+%s parsed the markup in %.2fs." % (parser, b-a) + + from lxml import etree + a = time.time() + etree.HTML(data) + b = time.time() + print "Raw lxml parsed the markup in %.2fs." % (b-a) + + import html5lib + parser = html5lib.HTMLParser() + a = time.time() + parser.parse(data) + b = time.time() + print "Raw html5lib parsed the markup in %.2fs." % (b-a) + +def profile(num_elements=100000, parser="lxml"): + + filehandle = tempfile.NamedTemporaryFile() + filename = filehandle.name + + data = rdoc(num_elements) + vars = dict(bs4=bs4, data=data, parser=parser) + cProfile.runctx('bs4.BeautifulSoup(data, parser)' , vars, vars, filename) + + stats = pstats.Stats(filename) + # stats.strip_dirs() + stats.sort_stats("cumulative") + stats.print_stats('_html5lib|bs4', 50) + +if __name__ == '__main__': + diagnose(sys.stdin.read()) diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/element.py b/import-layers/yocto-poky/bitbake/lib/bs4/element.py new file mode 100644 index 000000000..da9afdf48 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/element.py @@ -0,0 +1,1611 @@ +import collections +import re +import sys +import warnings +from bs4.dammit import EntitySubstitution + +DEFAULT_OUTPUT_ENCODING = "utf-8" +PY3K = (sys.version_info[0] > 2) + +whitespace_re = re.compile("\s+") + +def _alias(attr): + """Alias one attribute name to another for backward compatibility""" + @property + def alias(self): + return getattr(self, attr) + + @alias.setter + def alias(self): + return setattr(self, attr) + return alias + + +class NamespacedAttribute(unicode): + + def __new__(cls, prefix, name, namespace=None): + if name is None: + obj = unicode.__new__(cls, prefix) + elif prefix is None: + # Not really namespaced. + obj = unicode.__new__(cls, name) + else: + obj = unicode.__new__(cls, prefix + ":" + name) + obj.prefix = prefix + obj.name = name + obj.namespace = namespace + return obj + +class AttributeValueWithCharsetSubstitution(unicode): + """A stand-in object for a character encoding specified in HTML.""" + +class CharsetMetaAttributeValue(AttributeValueWithCharsetSubstitution): + """A generic stand-in for the value of a meta tag's 'charset' attribute. + + When Beautiful Soup parses the markup '<meta charset="utf8">', the + value of the 'charset' attribute will be one of these objects. + """ + + def __new__(cls, original_value): + obj = unicode.__new__(cls, original_value) + obj.original_value = original_value + return obj + + def encode(self, encoding): + return encoding + + +class ContentMetaAttributeValue(AttributeValueWithCharsetSubstitution): + """A generic stand-in for the value of a meta tag's 'content' attribute. + + When Beautiful Soup parses the markup: + <meta http-equiv="content-type" content="text/html; charset=utf8"> + + The value of the 'content' attribute will be one of these objects. + """ + + CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)", re.M) + + def __new__(cls, original_value): + match = cls.CHARSET_RE.search(original_value) + if match is None: + # No substitution necessary. + return unicode.__new__(unicode, original_value) + + obj = unicode.__new__(cls, original_value) + obj.original_value = original_value + return obj + + def encode(self, encoding): + def rewrite(match): + return match.group(1) + encoding + return self.CHARSET_RE.sub(rewrite, self.original_value) + +class HTMLAwareEntitySubstitution(EntitySubstitution): + + """Entity substitution rules that are aware of some HTML quirks. + + Specifically, the contents of <script> and <style> tags should not + undergo entity substitution. + + Incoming NavigableString objects are checked to see if they're the + direct children of a <script> or <style> tag. + """ + + cdata_containing_tags = set(["script", "style"]) + + preformatted_tags = set(["pre"]) + + @classmethod + def _substitute_if_appropriate(cls, ns, f): + if (isinstance(ns, NavigableString) + and ns.parent is not None + and ns.parent.name in cls.cdata_containing_tags): + # Do nothing. + return ns + # Substitute. + return f(ns) + + @classmethod + def substitute_html(cls, ns): + return cls._substitute_if_appropriate( + ns, EntitySubstitution.substitute_html) + + @classmethod + def substitute_xml(cls, ns): + return cls._substitute_if_appropriate( + ns, EntitySubstitution.substitute_xml) + +class PageElement(object): + """Contains the navigational information for some part of the page + (either a tag or a piece of text)""" + + # There are five possible values for the "formatter" argument passed in + # to methods like encode() and prettify(): + # + # "html" - All Unicode characters with corresponding HTML entities + # are converted to those entities on output. + # "minimal" - Bare ampersands and angle brackets are converted to + # XML entities: & < > + # None - The null formatter. Unicode characters are never + # converted to entities. This is not recommended, but it's + # faster than "minimal". + # A function - This function will be called on every string that + # needs to undergo entity substitution. + # + + # In an HTML document, the default "html" and "minimal" functions + # will leave the contents of <script> and <style> tags alone. For + # an XML document, all tags will be given the same treatment. + + HTML_FORMATTERS = { + "html" : HTMLAwareEntitySubstitution.substitute_html, + "minimal" : HTMLAwareEntitySubstitution.substitute_xml, + None : None + } + + XML_FORMATTERS = { + "html" : EntitySubstitution.substitute_html, + "minimal" : EntitySubstitution.substitute_xml, + None : None + } + + def format_string(self, s, formatter='minimal'): + """Format the given string using the given formatter.""" + if not callable(formatter): + formatter = self._formatter_for_name(formatter) + if formatter is None: + output = s + else: + output = formatter(s) + return output + + @property + def _is_xml(self): + """Is this element part of an XML tree or an HTML tree? + + This is used when mapping a formatter name ("minimal") to an + appropriate function (one that performs entity-substitution on + the contents of <script> and <style> tags, or not). It's + inefficient, but it should be called very rarely. + """ + if self.parent is None: + # This is the top-level object. It should have .is_xml set + # from tree creation. If not, take a guess--BS is usually + # used on HTML markup. + return getattr(self, 'is_xml', False) + return self.parent._is_xml + + def _formatter_for_name(self, name): + "Look up a formatter function based on its name and the tree." + if self._is_xml: + return self.XML_FORMATTERS.get( + name, EntitySubstitution.substitute_xml) + else: + return self.HTML_FORMATTERS.get( + name, HTMLAwareEntitySubstitution.substitute_xml) + + def setup(self, parent=None, previous_element=None): + """Sets up the initial relations between this element and + other elements.""" + self.parent = parent + self.previous_element = previous_element + if previous_element is not None: + self.previous_element.next_element = self + self.next_element = None + self.previous_sibling = None + self.next_sibling = None + if self.parent is not None and self.parent.contents: + self.previous_sibling = self.parent.contents[-1] + self.previous_sibling.next_sibling = self + + nextSibling = _alias("next_sibling") # BS3 + previousSibling = _alias("previous_sibling") # BS3 + + def replace_with(self, replace_with): + if replace_with is self: + return + if replace_with is self.parent: + raise ValueError("Cannot replace a Tag with its parent.") + old_parent = self.parent + my_index = self.parent.index(self) + self.extract() + old_parent.insert(my_index, replace_with) + return self + replaceWith = replace_with # BS3 + + def unwrap(self): + my_parent = self.parent + my_index = self.parent.index(self) + self.extract() + for child in reversed(self.contents[:]): + my_parent.insert(my_index, child) + return self + replace_with_children = unwrap + replaceWithChildren = unwrap # BS3 + + def wrap(self, wrap_inside): + me = self.replace_with(wrap_inside) + wrap_inside.append(me) + return wrap_inside + + def extract(self): + """Destructively rips this element out of the tree.""" + if self.parent is not None: + del self.parent.contents[self.parent.index(self)] + + #Find the two elements that would be next to each other if + #this element (and any children) hadn't been parsed. Connect + #the two. + last_child = self._last_descendant() + next_element = last_child.next_element + + if self.previous_element is not None: + self.previous_element.next_element = next_element + if next_element is not None: + next_element.previous_element = self.previous_element + self.previous_element = None + last_child.next_element = None + + self.parent = None + if self.previous_sibling is not None: + self.previous_sibling.next_sibling = self.next_sibling + if self.next_sibling is not None: + self.next_sibling.previous_sibling = self.previous_sibling + self.previous_sibling = self.next_sibling = None + return self + + def _last_descendant(self, is_initialized=True, accept_self=True): + "Finds the last element beneath this object to be parsed." + if is_initialized and self.next_sibling: + last_child = self.next_sibling.previous_element + else: + last_child = self + while isinstance(last_child, Tag) and last_child.contents: + last_child = last_child.contents[-1] + if not accept_self and last_child == self: + last_child = None + return last_child + # BS3: Not part of the API! + _lastRecursiveChild = _last_descendant + + def insert(self, position, new_child): + if new_child is self: + raise ValueError("Cannot insert a tag into itself.") + if (isinstance(new_child, basestring) + and not isinstance(new_child, NavigableString)): + new_child = NavigableString(new_child) + + position = min(position, len(self.contents)) + if hasattr(new_child, 'parent') and new_child.parent is not None: + # We're 'inserting' an element that's already one + # of this object's children. + if new_child.parent is self: + current_index = self.index(new_child) + if current_index < position: + # We're moving this element further down the list + # of this object's children. That means that when + # we extract this element, our target index will + # jump down one. + position -= 1 + new_child.extract() + + new_child.parent = self + previous_child = None + if position == 0: + new_child.previous_sibling = None + new_child.previous_element = self + else: + previous_child = self.contents[position - 1] + new_child.previous_sibling = previous_child + new_child.previous_sibling.next_sibling = new_child + new_child.previous_element = previous_child._last_descendant(False) + if new_child.previous_element is not None: + new_child.previous_element.next_element = new_child + + new_childs_last_element = new_child._last_descendant(False) + + if position >= len(self.contents): + new_child.next_sibling = None + + parent = self + parents_next_sibling = None + while parents_next_sibling is None and parent is not None: + parents_next_sibling = parent.next_sibling + parent = parent.parent + if parents_next_sibling is not None: + # We found the element that comes next in the document. + break + if parents_next_sibling is not None: + new_childs_last_element.next_element = parents_next_sibling + else: + # The last element of this tag is the last element in + # the document. + new_childs_last_element.next_element = None + else: + next_child = self.contents[position] + new_child.next_sibling = next_child + if new_child.next_sibling is not None: + new_child.next_sibling.previous_sibling = new_child + new_childs_last_element.next_element = next_child + + if new_childs_last_element.next_element is not None: + new_childs_last_element.next_element.previous_element = new_childs_last_element + self.contents.insert(position, new_child) + + def append(self, tag): + """Appends the given tag to the contents of this tag.""" + self.insert(len(self.contents), tag) + + def insert_before(self, predecessor): + """Makes the given element the immediate predecessor of this one. + + The two elements will have the same parent, and the given element + will be immediately before this one. + """ + if self is predecessor: + raise ValueError("Can't insert an element before itself.") + parent = self.parent + if parent is None: + raise ValueError( + "Element has no parent, so 'before' has no meaning.") + # Extract first so that the index won't be screwed up if they + # are siblings. + if isinstance(predecessor, PageElement): + predecessor.extract() + index = parent.index(self) + parent.insert(index, predecessor) + + def insert_after(self, successor): + """Makes the given element the immediate successor of this one. + + The two elements will have the same parent, and the given element + will be immediately after this one. + """ + if self is successor: + raise ValueError("Can't insert an element after itself.") + parent = self.parent + if parent is None: + raise ValueError( + "Element has no parent, so 'after' has no meaning.") + # Extract first so that the index won't be screwed up if they + # are siblings. + if isinstance(successor, PageElement): + successor.extract() + index = parent.index(self) + parent.insert(index+1, successor) + + def find_next(self, name=None, attrs={}, text=None, **kwargs): + """Returns the first item that matches the given criteria and + appears after this Tag in the document.""" + return self._find_one(self.find_all_next, name, attrs, text, **kwargs) + findNext = find_next # BS3 + + def find_all_next(self, name=None, attrs={}, text=None, limit=None, + **kwargs): + """Returns all items that match the given criteria and appear + after this Tag in the document.""" + return self._find_all(name, attrs, text, limit, self.next_elements, + **kwargs) + findAllNext = find_all_next # BS3 + + def find_next_sibling(self, name=None, attrs={}, text=None, **kwargs): + """Returns the closest sibling to this Tag that matches the + given criteria and appears after this Tag in the document.""" + return self._find_one(self.find_next_siblings, name, attrs, text, + **kwargs) + findNextSibling = find_next_sibling # BS3 + + def find_next_siblings(self, name=None, attrs={}, text=None, limit=None, + **kwargs): + """Returns the siblings of this Tag that match the given + criteria and appear after this Tag in the document.""" + return self._find_all(name, attrs, text, limit, + self.next_siblings, **kwargs) + findNextSiblings = find_next_siblings # BS3 + fetchNextSiblings = find_next_siblings # BS2 + + def find_previous(self, name=None, attrs={}, text=None, **kwargs): + """Returns the first item that matches the given criteria and + appears before this Tag in the document.""" + return self._find_one( + self.find_all_previous, name, attrs, text, **kwargs) + findPrevious = find_previous # BS3 + + def find_all_previous(self, name=None, attrs={}, text=None, limit=None, + **kwargs): + """Returns all items that match the given criteria and appear + before this Tag in the document.""" + return self._find_all(name, attrs, text, limit, self.previous_elements, + **kwargs) + findAllPrevious = find_all_previous # BS3 + fetchPrevious = find_all_previous # BS2 + + def find_previous_sibling(self, name=None, attrs={}, text=None, **kwargs): + """Returns the closest sibling to this Tag that matches the + given criteria and appears before this Tag in the document.""" + return self._find_one(self.find_previous_siblings, name, attrs, text, + **kwargs) + findPreviousSibling = find_previous_sibling # BS3 + + def find_previous_siblings(self, name=None, attrs={}, text=None, + limit=None, **kwargs): + """Returns the siblings of this Tag that match the given + criteria and appear before this Tag in the document.""" + return self._find_all(name, attrs, text, limit, + self.previous_siblings, **kwargs) + findPreviousSiblings = find_previous_siblings # BS3 + fetchPreviousSiblings = find_previous_siblings # BS2 + + def find_parent(self, name=None, attrs={}, **kwargs): + """Returns the closest parent of this Tag that matches the given + criteria.""" + # NOTE: We can't use _find_one because findParents takes a different + # set of arguments. + r = None + l = self.find_parents(name, attrs, 1, **kwargs) + if l: + r = l[0] + return r + findParent = find_parent # BS3 + + def find_parents(self, name=None, attrs={}, limit=None, **kwargs): + """Returns the parents of this Tag that match the given + criteria.""" + + return self._find_all(name, attrs, None, limit, self.parents, + **kwargs) + findParents = find_parents # BS3 + fetchParents = find_parents # BS2 + + @property + def next(self): + return self.next_element + + @property + def previous(self): + return self.previous_element + + #These methods do the real heavy lifting. + + def _find_one(self, method, name, attrs, text, **kwargs): + r = None + l = method(name, attrs, text, 1, **kwargs) + if l: + r = l[0] + return r + + def _find_all(self, name, attrs, text, limit, generator, **kwargs): + "Iterates over a generator looking for things that match." + + if isinstance(name, SoupStrainer): + strainer = name + else: + strainer = SoupStrainer(name, attrs, text, **kwargs) + + if text is None and not limit and not attrs and not kwargs: + if name is True or name is None: + # Optimization to find all tags. + result = (element for element in generator + if isinstance(element, Tag)) + return ResultSet(strainer, result) + elif isinstance(name, basestring): + # Optimization to find all tags with a given name. + result = (element for element in generator + if isinstance(element, Tag) + and element.name == name) + return ResultSet(strainer, result) + results = ResultSet(strainer) + while True: + try: + i = next(generator) + except StopIteration: + break + if i: + found = strainer.search(i) + if found: + results.append(found) + if limit and len(results) >= limit: + break + return results + + #These generators can be used to navigate starting from both + #NavigableStrings and Tags. + @property + def next_elements(self): + i = self.next_element + while i is not None: + yield i + i = i.next_element + + @property + def next_siblings(self): + i = self.next_sibling + while i is not None: + yield i + i = i.next_sibling + + @property + def previous_elements(self): + i = self.previous_element + while i is not None: + yield i + i = i.previous_element + + @property + def previous_siblings(self): + i = self.previous_sibling + while i is not None: + yield i + i = i.previous_sibling + + @property + def parents(self): + i = self.parent + while i is not None: + yield i + i = i.parent + + # Methods for supporting CSS selectors. + + tag_name_re = re.compile('^[a-z0-9]+$') + + # /^(\w+)\[(\w+)([=~\|\^\$\*]?)=?"?([^\]"]*)"?\]$/ + # \---/ \---/\-------------/ \-------/ + # | | | | + # | | | The value + # | | ~,|,^,$,* or = + # | Attribute + # Tag + attribselect_re = re.compile( + r'^(?P<tag>\w+)?\[(?P<attribute>\w+)(?P<operator>[=~\|\^\$\*]?)' + + r'=?"?(?P<value>[^\]"]*)"?\]$' + ) + + def _attr_value_as_string(self, value, default=None): + """Force an attribute value into a string representation. + + A multi-valued attribute will be converted into a + space-separated stirng. + """ + value = self.get(value, default) + if isinstance(value, list) or isinstance(value, tuple): + value =" ".join(value) + return value + + def _tag_name_matches_and(self, function, tag_name): + if not tag_name: + return function + else: + def _match(tag): + return tag.name == tag_name and function(tag) + return _match + + def _attribute_checker(self, operator, attribute, value=''): + """Create a function that performs a CSS selector operation. + + Takes an operator, attribute and optional value. Returns a + function that will return True for elements that match that + combination. + """ + if operator == '=': + # string representation of `attribute` is equal to `value` + return lambda el: el._attr_value_as_string(attribute) == value + elif operator == '~': + # space-separated list representation of `attribute` + # contains `value` + def _includes_value(element): + attribute_value = element.get(attribute, []) + if not isinstance(attribute_value, list): + attribute_value = attribute_value.split() + return value in attribute_value + return _includes_value + elif operator == '^': + # string representation of `attribute` starts with `value` + return lambda el: el._attr_value_as_string( + attribute, '').startswith(value) + elif operator == '$': + # string represenation of `attribute` ends with `value` + return lambda el: el._attr_value_as_string( + attribute, '').endswith(value) + elif operator == '*': + # string representation of `attribute` contains `value` + return lambda el: value in el._attr_value_as_string(attribute, '') + elif operator == '|': + # string representation of `attribute` is either exactly + # `value` or starts with `value` and then a dash. + def _is_or_starts_with_dash(element): + attribute_value = element._attr_value_as_string(attribute, '') + return (attribute_value == value or attribute_value.startswith( + value + '-')) + return _is_or_starts_with_dash + else: + return lambda el: el.has_attr(attribute) + + # Old non-property versions of the generators, for backwards + # compatibility with BS3. + def nextGenerator(self): + return self.next_elements + + def nextSiblingGenerator(self): + return self.next_siblings + + def previousGenerator(self): + return self.previous_elements + + def previousSiblingGenerator(self): + return self.previous_siblings + + def parentGenerator(self): + return self.parents + + +class NavigableString(unicode, PageElement): + + PREFIX = '' + SUFFIX = '' + + def __new__(cls, value): + """Create a new NavigableString. + + When unpickling a NavigableString, this method is called with + the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be + passed in to the superclass's __new__ or the superclass won't know + how to handle non-ASCII characters. + """ + if isinstance(value, unicode): + return unicode.__new__(cls, value) + return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING) + + def __copy__(self): + return self + + def __getnewargs__(self): + return (unicode(self),) + + def __getattr__(self, attr): + """text.string gives you text. This is for backwards + compatibility for Navigable*String, but for CData* it lets you + get the string without the CData wrapper.""" + if attr == 'string': + return self + else: + raise AttributeError( + "'%s' object has no attribute '%s'" % ( + self.__class__.__name__, attr)) + + def output_ready(self, formatter="minimal"): + output = self.format_string(self, formatter) + return self.PREFIX + output + self.SUFFIX + + @property + def name(self): + return None + + @name.setter + def name(self, name): + raise AttributeError("A NavigableString cannot be given a name.") + +class PreformattedString(NavigableString): + """A NavigableString not subject to the normal formatting rules. + + The string will be passed into the formatter (to trigger side effects), + but the return value will be ignored. + """ + + def output_ready(self, formatter="minimal"): + """CData strings are passed into the formatter. + But the return value is ignored.""" + self.format_string(self, formatter) + return self.PREFIX + self + self.SUFFIX + +class CData(PreformattedString): + + PREFIX = u'<![CDATA[' + SUFFIX = u']]>' + +class ProcessingInstruction(PreformattedString): + + PREFIX = u'<?' + SUFFIX = u'?>' + +class Comment(PreformattedString): + + PREFIX = u'<!--' + SUFFIX = u'-->' + + +class Declaration(PreformattedString): + PREFIX = u'<!' + SUFFIX = u'!>' + + +class Doctype(PreformattedString): + + @classmethod + def for_name_and_ids(cls, name, pub_id, system_id): + value = name or '' + if pub_id is not None: + value += ' PUBLIC "%s"' % pub_id + if system_id is not None: + value += ' "%s"' % system_id + elif system_id is not None: + value += ' SYSTEM "%s"' % system_id + + return Doctype(value) + + PREFIX = u'<!DOCTYPE ' + SUFFIX = u'>\n' + + +class Tag(PageElement): + + """Represents a found HTML tag with its attributes and contents.""" + + def __init__(self, parser=None, builder=None, name=None, namespace=None, + prefix=None, attrs=None, parent=None, previous=None): + "Basic constructor." + + if parser is None: + self.parser_class = None + else: + # We don't actually store the parser object: that lets extracted + # chunks be garbage-collected. + self.parser_class = parser.__class__ + if name is None: + raise ValueError("No value provided for new tag's name.") + self.name = name + self.namespace = namespace + self.prefix = prefix + if attrs is None: + attrs = {} + elif attrs and builder.cdata_list_attributes: + attrs = builder._replace_cdata_list_attribute_values( + self.name, attrs) + else: + attrs = dict(attrs) + self.attrs = attrs + self.contents = [] + self.setup(parent, previous) + self.hidden = False + + # Set up any substitutions, such as the charset in a META tag. + if builder is not None: + builder.set_up_substitutions(self) + self.can_be_empty_element = builder.can_be_empty_element(name) + else: + self.can_be_empty_element = False + + parserClass = _alias("parser_class") # BS3 + + @property + def is_empty_element(self): + """Is this tag an empty-element tag? (aka a self-closing tag) + + A tag that has contents is never an empty-element tag. + + A tag that has no contents may or may not be an empty-element + tag. It depends on the builder used to create the tag. If the + builder has a designated list of empty-element tags, then only + a tag whose name shows up in that list is considered an + empty-element tag. + + If the builder has no designated list of empty-element tags, + then any tag with no contents is an empty-element tag. + """ + return len(self.contents) == 0 and self.can_be_empty_element + isSelfClosing = is_empty_element # BS3 + + @property + def string(self): + """Convenience property to get the single string within this tag. + + :Return: If this tag has a single string child, return value + is that string. If this tag has no children, or more than one + child, return value is None. If this tag has one child tag, + return value is the 'string' attribute of the child tag, + recursively. + """ + if len(self.contents) != 1: + return None + child = self.contents[0] + if isinstance(child, NavigableString): + return child + return child.string + + @string.setter + def string(self, string): + self.clear() + self.append(string.__class__(string)) + + def _all_strings(self, strip=False, types=(NavigableString, CData)): + """Yield all strings of certain classes, possibly stripping them. + + By default, yields only NavigableString and CData objects. So + no comments, processing instructions, etc. + """ + for descendant in self.descendants: + if ( + (types is None and not isinstance(descendant, NavigableString)) + or + (types is not None and type(descendant) not in types)): + continue + if strip: + descendant = descendant.strip() + if len(descendant) == 0: + continue + yield descendant + + strings = property(_all_strings) + + @property + def stripped_strings(self): + for string in self._all_strings(True): + yield string + + def get_text(self, separator=u"", strip=False, + types=(NavigableString, CData)): + """ + Get all child strings, concatenated using the given separator. + """ + return separator.join([s for s in self._all_strings( + strip, types=types)]) + getText = get_text + text = property(get_text) + + def decompose(self): + """Recursively destroys the contents of this tree.""" + self.extract() + i = self + while i is not None: + next = i.next_element + i.__dict__.clear() + i.contents = [] + i = next + + def clear(self, decompose=False): + """ + Extract all children. If decompose is True, decompose instead. + """ + if decompose: + for element in self.contents[:]: + if isinstance(element, Tag): + element.decompose() + else: + element.extract() + else: + for element in self.contents[:]: + element.extract() + + def index(self, element): + """ + Find the index of a child by identity, not value. Avoids issues with + tag.contents.index(element) getting the index of equal elements. + """ + for i, child in enumerate(self.contents): + if child is element: + return i + raise ValueError("Tag.index: element not in tag") + + def get(self, key, default=None): + """Returns the value of the 'key' attribute for the tag, or + the value given for 'default' if it doesn't have that + attribute.""" + return self.attrs.get(key, default) + + def has_attr(self, key): + return key in self.attrs + + def __hash__(self): + return str(self).__hash__() + + def __getitem__(self, key): + """tag[key] returns the value of the 'key' attribute for the tag, + and throws an exception if it's not there.""" + return self.attrs[key] + + def __iter__(self): + "Iterating over a tag iterates over its contents." + return iter(self.contents) + + def __len__(self): + "The length of a tag is the length of its list of contents." + return len(self.contents) + + def __contains__(self, x): + return x in self.contents + + def __nonzero__(self): + "A tag is non-None even if it has no contents." + return True + + def __setitem__(self, key, value): + """Setting tag[key] sets the value of the 'key' attribute for the + tag.""" + self.attrs[key] = value + + def __delitem__(self, key): + "Deleting tag[key] deletes all 'key' attributes for the tag." + self.attrs.pop(key, None) + + def __call__(self, *args, **kwargs): + """Calling a tag like a function is the same as calling its + find_all() method. Eg. tag('a') returns a list of all the A tags + found within this tag.""" + return self.find_all(*args, **kwargs) + + def __getattr__(self, tag): + #print "Getattr %s.%s" % (self.__class__, tag) + if len(tag) > 3 and tag.endswith('Tag'): + # BS3: soup.aTag -> "soup.find("a") + tag_name = tag[:-3] + warnings.warn( + '.%sTag is deprecated, use .find("%s") instead.' % ( + tag_name, tag_name)) + return self.find(tag_name) + # We special case contents to avoid recursion. + elif not tag.startswith("__") and not tag=="contents": + return self.find(tag) + raise AttributeError( + "'%s' object has no attribute '%s'" % (self.__class__, tag)) + + def __eq__(self, other): + """Returns true iff this tag has the same name, the same attributes, + and the same contents (recursively) as the given tag.""" + if self is other: + return True + if (not hasattr(other, 'name') or + not hasattr(other, 'attrs') or + not hasattr(other, 'contents') or + self.name != other.name or + self.attrs != other.attrs or + len(self) != len(other)): + return False + for i, my_child in enumerate(self.contents): + if my_child != other.contents[i]: + return False + return True + + def __ne__(self, other): + """Returns true iff this tag is not identical to the other tag, + as defined in __eq__.""" + return not self == other + + def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING): + """Renders this tag as a string.""" + return self.encode(encoding) + + def __unicode__(self): + return self.decode() + + def __str__(self): + return self.encode() + + if PY3K: + __str__ = __repr__ = __unicode__ + + def encode(self, encoding=DEFAULT_OUTPUT_ENCODING, + indent_level=None, formatter="minimal", + errors="xmlcharrefreplace"): + # Turn the data structure into Unicode, then encode the + # Unicode. + u = self.decode(indent_level, encoding, formatter) + return u.encode(encoding, errors) + + def _should_pretty_print(self, indent_level): + """Should this tag be pretty-printed?""" + return ( + indent_level is not None and + (self.name not in HTMLAwareEntitySubstitution.preformatted_tags + or self._is_xml)) + + def decode(self, indent_level=None, + eventual_encoding=DEFAULT_OUTPUT_ENCODING, + formatter="minimal"): + """Returns a Unicode representation of this tag and its contents. + + :param eventual_encoding: The tag is destined to be + encoded into this encoding. This method is _not_ + responsible for performing that encoding. This information + is passed in so that it can be substituted in if the + document contains a <META> tag that mentions the document's + encoding. + """ + + # First off, turn a string formatter into a function. This + # will stop the lookup from happening over and over again. + if not callable(formatter): + formatter = self._formatter_for_name(formatter) + + attrs = [] + if self.attrs: + for key, val in sorted(self.attrs.items()): + if val is None: + decoded = key + else: + if isinstance(val, list) or isinstance(val, tuple): + val = ' '.join(val) + elif not isinstance(val, basestring): + val = unicode(val) + elif ( + isinstance(val, AttributeValueWithCharsetSubstitution) + and eventual_encoding is not None): + val = val.encode(eventual_encoding) + + text = self.format_string(val, formatter) + decoded = ( + unicode(key) + '=' + + EntitySubstitution.quoted_attribute_value(text)) + attrs.append(decoded) + close = '' + closeTag = '' + + prefix = '' + if self.prefix: + prefix = self.prefix + ":" + + if self.is_empty_element: + close = '/' + else: + closeTag = '</%s%s>' % (prefix, self.name) + + pretty_print = self._should_pretty_print(indent_level) + space = '' + indent_space = '' + if indent_level is not None: + indent_space = (' ' * (indent_level - 1)) + if pretty_print: + space = indent_space + indent_contents = indent_level + 1 + else: + indent_contents = None + contents = self.decode_contents( + indent_contents, eventual_encoding, formatter) + + if self.hidden: + # This is the 'document root' object. + s = contents + else: + s = [] + attribute_string = '' + if attrs: + attribute_string = ' ' + ' '.join(attrs) + if indent_level is not None: + # Even if this particular tag is not pretty-printed, + # we should indent up to the start of the tag. + s.append(indent_space) + s.append('<%s%s%s%s>' % ( + prefix, self.name, attribute_string, close)) + if pretty_print: + s.append("\n") + s.append(contents) + if pretty_print and contents and contents[-1] != "\n": + s.append("\n") + if pretty_print and closeTag: + s.append(space) + s.append(closeTag) + if indent_level is not None and closeTag and self.next_sibling: + # Even if this particular tag is not pretty-printed, + # we're now done with the tag, and we should add a + # newline if appropriate. + s.append("\n") + s = ''.join(s) + return s + + def prettify(self, encoding=None, formatter="minimal"): + if encoding is None: + return self.decode(True, formatter=formatter) + else: + return self.encode(encoding, True, formatter=formatter) + + def decode_contents(self, indent_level=None, + eventual_encoding=DEFAULT_OUTPUT_ENCODING, + formatter="minimal"): + """Renders the contents of this tag as a Unicode string. + + :param eventual_encoding: The tag is destined to be + encoded into this encoding. This method is _not_ + responsible for performing that encoding. This information + is passed in so that it can be substituted in if the + document contains a <META> tag that mentions the document's + encoding. + """ + # First off, turn a string formatter into a function. This + # will stop the lookup from happening over and over again. + if not callable(formatter): + formatter = self._formatter_for_name(formatter) + + pretty_print = (indent_level is not None) + s = [] + for c in self: + text = None + if isinstance(c, NavigableString): + text = c.output_ready(formatter) + elif isinstance(c, Tag): + s.append(c.decode(indent_level, eventual_encoding, + formatter)) + if text and indent_level and not self.name == 'pre': + text = text.strip() + if text: + if pretty_print and not self.name == 'pre': + s.append(" " * (indent_level - 1)) + s.append(text) + if pretty_print and not self.name == 'pre': + s.append("\n") + return ''.join(s) + + def encode_contents( + self, indent_level=None, encoding=DEFAULT_OUTPUT_ENCODING, + formatter="minimal"): + """Renders the contents of this tag as a bytestring.""" + contents = self.decode_contents(indent_level, encoding, formatter) + return contents.encode(encoding) + + # Old method for BS3 compatibility + def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING, + prettyPrint=False, indentLevel=0): + if not prettyPrint: + indentLevel = None + return self.encode_contents( + indent_level=indentLevel, encoding=encoding) + + #Soup methods + + def find(self, name=None, attrs={}, recursive=True, text=None, + **kwargs): + """Return only the first child of this Tag matching the given + criteria.""" + r = None + l = self.find_all(name, attrs, recursive, text, 1, **kwargs) + if l: + r = l[0] + return r + findChild = find + + def find_all(self, name=None, attrs={}, recursive=True, text=None, + limit=None, **kwargs): + """Extracts a list of Tag objects that match the given + criteria. You can specify the name of the Tag and any + attributes you want the Tag to have. + + The value of a key-value pair in the 'attrs' map can be a + string, a list of strings, a regular expression object, or a + callable that takes a string and returns whether or not the + string matches for some custom definition of 'matches'. The + same is true of the tag name.""" + + generator = self.descendants + if not recursive: + generator = self.children + return self._find_all(name, attrs, text, limit, generator, **kwargs) + findAll = find_all # BS3 + findChildren = find_all # BS2 + + #Generator methods + @property + def children(self): + # return iter() to make the purpose of the method clear + return iter(self.contents) # XXX This seems to be untested. + + @property + def descendants(self): + if not len(self.contents): + return + stopNode = self._last_descendant().next_element + current = self.contents[0] + while current is not stopNode: + yield current + current = current.next_element + + # CSS selector code + + _selector_combinators = ['>', '+', '~'] + _select_debug = False + def select(self, selector, _candidate_generator=None): + """Perform a CSS selection operation on the current element.""" + tokens = selector.split() + current_context = [self] + + if tokens[-1] in self._selector_combinators: + raise ValueError( + 'Final combinator "%s" is missing an argument.' % tokens[-1]) + if self._select_debug: + print 'Running CSS selector "%s"' % selector + for index, token in enumerate(tokens): + if self._select_debug: + print ' Considering token "%s"' % token + recursive_candidate_generator = None + tag_name = None + if tokens[index-1] in self._selector_combinators: + # This token was consumed by the previous combinator. Skip it. + if self._select_debug: + print ' Token was consumed by the previous combinator.' + continue + # Each operation corresponds to a checker function, a rule + # for determining whether a candidate matches the + # selector. Candidates are generated by the active + # iterator. + checker = None + + m = self.attribselect_re.match(token) + if m is not None: + # Attribute selector + tag_name, attribute, operator, value = m.groups() + checker = self._attribute_checker(operator, attribute, value) + + elif '#' in token: + # ID selector + tag_name, tag_id = token.split('#', 1) + def id_matches(tag): + return tag.get('id', None) == tag_id + checker = id_matches + + elif '.' in token: + # Class selector + tag_name, klass = token.split('.', 1) + classes = set(klass.split('.')) + def classes_match(candidate): + return classes.issubset(candidate.get('class', [])) + checker = classes_match + + elif ':' in token: + # Pseudo-class + tag_name, pseudo = token.split(':', 1) + if tag_name == '': + raise ValueError( + "A pseudo-class must be prefixed with a tag name.") + pseudo_attributes = re.match('([a-zA-Z\d-]+)\(([a-zA-Z\d]+)\)', pseudo) + found = [] + if pseudo_attributes is not None: + pseudo_type, pseudo_value = pseudo_attributes.groups() + if pseudo_type == 'nth-of-type': + try: + pseudo_value = int(pseudo_value) + except: + raise NotImplementedError( + 'Only numeric values are currently supported for the nth-of-type pseudo-class.') + if pseudo_value < 1: + raise ValueError( + 'nth-of-type pseudo-class value must be at least 1.') + class Counter(object): + def __init__(self, destination): + self.count = 0 + self.destination = destination + + def nth_child_of_type(self, tag): + self.count += 1 + if self.count == self.destination: + return True + if self.count > self.destination: + # Stop the generator that's sending us + # these things. + raise StopIteration() + return False + checker = Counter(pseudo_value).nth_child_of_type + else: + raise NotImplementedError( + 'Only the following pseudo-classes are implemented: nth-of-type.') + + elif token == '*': + # Star selector -- matches everything + pass + elif token == '>': + # Run the next token as a CSS selector against the + # direct children of each tag in the current context. + recursive_candidate_generator = lambda tag: tag.children + elif token == '~': + # Run the next token as a CSS selector against the + # siblings of each tag in the current context. + recursive_candidate_generator = lambda tag: tag.next_siblings + elif token == '+': + # For each tag in the current context, run the next + # token as a CSS selector against the tag's next + # sibling that's a tag. + def next_tag_sibling(tag): + yield tag.find_next_sibling(True) + recursive_candidate_generator = next_tag_sibling + + elif self.tag_name_re.match(token): + # Just a tag name. + tag_name = token + else: + raise ValueError( + 'Unsupported or invalid CSS selector: "%s"' % token) + + if recursive_candidate_generator: + # This happens when the selector looks like "> foo". + # + # The generator calls select() recursively on every + # member of the current context, passing in a different + # candidate generator and a different selector. + # + # In the case of "> foo", the candidate generator is + # one that yields a tag's direct children (">"), and + # the selector is "foo". + next_token = tokens[index+1] + def recursive_select(tag): + if self._select_debug: + print ' Calling select("%s") recursively on %s %s' % (next_token, tag.name, tag.attrs) + print '-' * 40 + for i in tag.select(next_token, recursive_candidate_generator): + if self._select_debug: + print '(Recursive select picked up candidate %s %s)' % (i.name, i.attrs) + yield i + if self._select_debug: + print '-' * 40 + _use_candidate_generator = recursive_select + elif _candidate_generator is None: + # By default, a tag's candidates are all of its + # children. If tag_name is defined, only yield tags + # with that name. + if self._select_debug: + if tag_name: + check = "[any]" + else: + check = tag_name + print ' Default candidate generator, tag name="%s"' % check + if self._select_debug: + # This is redundant with later code, but it stops + # a bunch of bogus tags from cluttering up the + # debug log. + def default_candidate_generator(tag): + for child in tag.descendants: + if not isinstance(child, Tag): + continue + if tag_name and not child.name == tag_name: + continue + yield child + _use_candidate_generator = default_candidate_generator + else: + _use_candidate_generator = lambda tag: tag.descendants + else: + _use_candidate_generator = _candidate_generator + + new_context = [] + new_context_ids = set([]) + for tag in current_context: + if self._select_debug: + print " Running candidate generator on %s %s" % ( + tag.name, repr(tag.attrs)) + for candidate in _use_candidate_generator(tag): + if not isinstance(candidate, Tag): + continue + if tag_name and candidate.name != tag_name: + continue + if checker is not None: + try: + result = checker(candidate) + except StopIteration: + # The checker has decided we should no longer + # run the generator. + break + if checker is None or result: + if self._select_debug: + print " SUCCESS %s %s" % (candidate.name, repr(candidate.attrs)) + if id(candidate) not in new_context_ids: + # If a tag matches a selector more than once, + # don't include it in the context more than once. + new_context.append(candidate) + new_context_ids.add(id(candidate)) + elif self._select_debug: + print " FAILURE %s %s" % (candidate.name, repr(candidate.attrs)) + + current_context = new_context + + if self._select_debug: + print "Final verdict:" + for i in current_context: + print " %s %s" % (i.name, i.attrs) + return current_context + + # Old names for backwards compatibility + def childGenerator(self): + return self.children + + def recursiveChildGenerator(self): + return self.descendants + + def has_key(self, key): + """This was kind of misleading because has_key() (attributes) + was different from __in__ (contents). has_key() is gone in + Python 3, anyway.""" + warnings.warn('has_key is deprecated. Use has_attr("%s") instead.' % ( + key)) + return self.has_attr(key) + +# Next, a couple classes to represent queries and their results. +class SoupStrainer(object): + """Encapsulates a number of ways of matching a markup element (tag or + text).""" + + def __init__(self, name=None, attrs={}, text=None, **kwargs): + self.name = self._normalize_search_value(name) + if not isinstance(attrs, dict): + # Treat a non-dict value for attrs as a search for the 'class' + # attribute. + kwargs['class'] = attrs + attrs = None + + if 'class_' in kwargs: + # Treat class_="foo" as a search for the 'class' + # attribute, overriding any non-dict value for attrs. + kwargs['class'] = kwargs['class_'] + del kwargs['class_'] + + if kwargs: + if attrs: + attrs = attrs.copy() + attrs.update(kwargs) + else: + attrs = kwargs + normalized_attrs = {} + for key, value in attrs.items(): + normalized_attrs[key] = self._normalize_search_value(value) + + self.attrs = normalized_attrs + self.text = self._normalize_search_value(text) + + def _normalize_search_value(self, value): + # Leave it alone if it's a Unicode string, a callable, a + # regular expression, a boolean, or None. + if (isinstance(value, unicode) or callable(value) or hasattr(value, 'match') + or isinstance(value, bool) or value is None): + return value + + # If it's a bytestring, convert it to Unicode, treating it as UTF-8. + if isinstance(value, bytes): + return value.decode("utf8") + + # If it's listlike, convert it into a list of strings. + if hasattr(value, '__iter__'): + new_value = [] + for v in value: + if (hasattr(v, '__iter__') and not isinstance(v, bytes) + and not isinstance(v, unicode)): + # This is almost certainly the user's mistake. In the + # interests of avoiding infinite loops, we'll let + # it through as-is rather than doing a recursive call. + new_value.append(v) + else: + new_value.append(self._normalize_search_value(v)) + return new_value + + # Otherwise, convert it into a Unicode string. + # The unicode(str()) thing is so this will do the same thing on Python 2 + # and Python 3. + return unicode(str(value)) + + def __str__(self): + if self.text: + return self.text + else: + return "%s|%s" % (self.name, self.attrs) + + def search_tag(self, markup_name=None, markup_attrs={}): + found = None + markup = None + if isinstance(markup_name, Tag): + markup = markup_name + markup_attrs = markup + call_function_with_tag_data = ( + isinstance(self.name, collections.Callable) + and not isinstance(markup_name, Tag)) + + if ((not self.name) + or call_function_with_tag_data + or (markup and self._matches(markup, self.name)) + or (not markup and self._matches(markup_name, self.name))): + if call_function_with_tag_data: + match = self.name(markup_name, markup_attrs) + else: + match = True + markup_attr_map = None + for attr, match_against in list(self.attrs.items()): + if not markup_attr_map: + if hasattr(markup_attrs, 'get'): + markup_attr_map = markup_attrs + else: + markup_attr_map = {} + for k, v in markup_attrs: + markup_attr_map[k] = v + attr_value = markup_attr_map.get(attr) + if not self._matches(attr_value, match_against): + match = False + break + if match: + if markup: + found = markup + else: + found = markup_name + if found and self.text and not self._matches(found.string, self.text): + found = None + return found + searchTag = search_tag + + def search(self, markup): + # print 'looking for %s in %s' % (self, markup) + found = None + # If given a list of items, scan it for a text element that + # matches. + if hasattr(markup, '__iter__') and not isinstance(markup, (Tag, basestring)): + for element in markup: + if isinstance(element, NavigableString) \ + and self.search(element): + found = element + break + # If it's a Tag, make sure its name or attributes match. + # Don't bother with Tags if we're searching for text. + elif isinstance(markup, Tag): + if not self.text or self.name or self.attrs: + found = self.search_tag(markup) + # If it's text, make sure the text matches. + elif isinstance(markup, NavigableString) or \ + isinstance(markup, basestring): + if not self.name and not self.attrs and self._matches(markup, self.text): + found = markup + else: + raise Exception( + "I don't know how to match against a %s" % markup.__class__) + return found + + def _matches(self, markup, match_against): + # print u"Matching %s against %s" % (markup, match_against) + result = False + if isinstance(markup, list) or isinstance(markup, tuple): + # This should only happen when searching a multi-valued attribute + # like 'class'. + if (isinstance(match_against, unicode) + and ' ' in match_against): + # A bit of a special case. If they try to match "foo + # bar" on a multivalue attribute's value, only accept + # the literal value "foo bar" + # + # XXX This is going to be pretty slow because we keep + # splitting match_against. But it shouldn't come up + # too often. + return (whitespace_re.split(match_against) == markup) + else: + for item in markup: + if self._matches(item, match_against): + return True + return False + + if match_against is True: + # True matches any non-None value. + return markup is not None + + if isinstance(match_against, collections.Callable): + return match_against(markup) + + # Custom callables take the tag as an argument, but all + # other ways of matching match the tag name as a string. + if isinstance(markup, Tag): + markup = markup.name + + # Ensure that `markup` is either a Unicode string, or None. + markup = self._normalize_search_value(markup) + + if markup is None: + # None matches None, False, an empty string, an empty list, and so on. + return not match_against + + if isinstance(match_against, unicode): + # Exact string match + return markup == match_against + + if hasattr(match_against, 'match'): + # Regexp match + return match_against.search(markup) + + if hasattr(match_against, '__iter__'): + # The markup must be an exact match against something + # in the iterable. + return markup in match_against + + +class ResultSet(list): + """A ResultSet is just a list that keeps track of the SoupStrainer + that created it.""" + def __init__(self, source, result=()): + super(ResultSet, self).__init__(result) + self.source = source diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/testing.py b/import-layers/yocto-poky/bitbake/lib/bs4/testing.py new file mode 100644 index 000000000..fd4495ac5 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bs4/testing.py @@ -0,0 +1,592 @@ +"""Helper classes for tests.""" + +import copy +import functools +import unittest +from unittest import TestCase +from bs4 import BeautifulSoup +from bs4.element import ( + CharsetMetaAttributeValue, + Comment, + ContentMetaAttributeValue, + Doctype, + SoupStrainer, +) + +from bs4.builder import HTMLParserTreeBuilder +default_builder = HTMLParserTreeBuilder + + +class SoupTest(unittest.TestCase): + + @property + def default_builder(self): + return default_builder() + + def soup(self, markup, **kwargs): + """Build a Beautiful Soup object from markup.""" + builder = kwargs.pop('builder', self.default_builder) + return BeautifulSoup(markup, builder=builder, **kwargs) + + def document_for(self, markup): + """Turn an HTML fragment into a document. + + The details depend on the builder. + """ + return self.default_builder.test_fragment_to_document(markup) + + def assertSoupEquals(self, to_parse, compare_parsed_to=None): + builder = self.default_builder + obj = BeautifulSoup(to_parse, builder=builder) + if compare_parsed_to is None: + compare_parsed_to = to_parse + + self.assertEqual(obj.decode(), self.document_for(compare_parsed_to)) + + +class HTMLTreeBuilderSmokeTest(object): + + """A basic test of a treebuilder's competence. + + Any HTML treebuilder, present or future, should be able to pass + these tests. With invalid markup, there's room for interpretation, + and different parsers can handle it differently. But with the + markup in these tests, there's not much room for interpretation. + """ + + def assertDoctypeHandled(self, doctype_fragment): + """Assert that a given doctype string is handled correctly.""" + doctype_str, soup = self._document_with_doctype(doctype_fragment) + + # Make sure a Doctype object was created. + doctype = soup.contents[0] + self.assertEqual(doctype.__class__, Doctype) + self.assertEqual(doctype, doctype_fragment) + self.assertEqual(str(soup)[:len(doctype_str)], doctype_str) + + # Make sure that the doctype was correctly associated with the + # parse tree and that the rest of the document parsed. + self.assertEqual(soup.p.contents[0], 'foo') + + def _document_with_doctype(self, doctype_fragment): + """Generate and parse a document with the given doctype.""" + doctype = '<!DOCTYPE %s>' % doctype_fragment + markup = doctype + '\n<p>foo</p>' + soup = self.soup(markup) + return doctype, soup + + def test_normal_doctypes(self): + """Make sure normal, everyday HTML doctypes are handled correctly.""" + self.assertDoctypeHandled("html") + self.assertDoctypeHandled( + 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"') + + def test_empty_doctype(self): + soup = self.soup("<!DOCTYPE>") + doctype = soup.contents[0] + self.assertEqual("", doctype.strip()) + + def test_public_doctype_with_url(self): + doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"' + self.assertDoctypeHandled(doctype) + + def test_system_doctype(self): + self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"') + + def test_namespaced_system_doctype(self): + # We can handle a namespaced doctype with a system ID. + self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"') + + def test_namespaced_public_doctype(self): + # Test a namespaced doctype with a public id. + self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"') + + def test_real_xhtml_document(self): + """A real XHTML document should come out more or less the same as it went in.""" + markup = b"""<?xml version="1.0" encoding="utf-8"?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"> +<html xmlns="http://www.w3.org/1999/xhtml"> +<head><title>Hello. +Goodbye. +""" + soup = self.soup(markup) + self.assertEqual( + soup.encode("utf-8").replace(b"\n", b""), + markup.replace(b"\n", b"")) + + def test_deepcopy(self): + """Make sure you can copy the tree builder. + + This is important because the builder is part of a + BeautifulSoup object, and we want to be able to copy that. + """ + copy.deepcopy(self.default_builder) + + def test_p_tag_is_never_empty_element(self): + """A

tag is never designated as an empty-element tag. + + Even if the markup shows it as an empty-element tag, it + shouldn't be presented that way. + """ + soup = self.soup("

") + self.assertFalse(soup.p.is_empty_element) + self.assertEqual(str(soup.p), "

") + + def test_unclosed_tags_get_closed(self): + """A tag that's not closed by the end of the document should be closed. + + This applies to all tags except empty-element tags. + """ + self.assertSoupEquals("

", "

") + self.assertSoupEquals("", "") + + self.assertSoupEquals("
", "
") + + def test_br_is_always_empty_element_tag(self): + """A
tag is designated as an empty-element tag. + + Some parsers treat

as one
tag, some parsers as + two tags, but it should always be an empty-element tag. + """ + soup = self.soup("

") + self.assertTrue(soup.br.is_empty_element) + self.assertEqual(str(soup.br), "
") + + def test_nested_formatting_elements(self): + self.assertSoupEquals("") + + def test_comment(self): + # Comments are represented as Comment objects. + markup = "

foobaz

" + self.assertSoupEquals(markup) + + soup = self.soup(markup) + comment = soup.find(text="foobar") + self.assertEqual(comment.__class__, Comment) + + # The comment is properly integrated into the tree. + foo = soup.find(text="foo") + self.assertEqual(comment, foo.next_element) + baz = soup.find(text="baz") + self.assertEqual(comment, baz.previous_element) + + def test_preserved_whitespace_in_pre_and_textarea(self): + """Whitespace must be preserved in 
 and ")
+
+    def test_nested_inline_elements(self):
+        """Inline elements can be nested indefinitely."""
+        b_tag = "Inside a B tag"
+        self.assertSoupEquals(b_tag)
+
+        nested_b_tag = "
A nested tag
"
+        self.assertSoupEquals(nested_b_tag)
+
+        double_nested_b_tag = "
A doubly nested tag
"
+        self.assertSoupEquals(nested_b_tag)
+
+    def test_nested_block_level_elements(self):
+        """Block elements can be nested."""
+        soup = self.soup('
Foo
')
+        blockquote = soup.blockquote
+        self.assertEqual(blockquote.p.b.string, 'Foo')
+        self.assertEqual(blockquote.b.string, 'Foo')
+
+    def test_correctly_nested_tables(self):
+        """One table can go inside another one."""
+        markup = (''
+                  ''
+                  "')
+
+        self.assertSoupEquals(
+            markup,
+            '
Here's another table:"
+                  ''
+                  ''
+                  '
foo
Here\'s another table:'
+            '
foo
'
+            '
')
+
+        self.assertSoupEquals(
+            ""
+            ""
+            "
Foo
Bar
Baz
")
+
+    def test_deeply_nested_multivalued_attribute(self):
+        # html5lib can set the attributes of the same tag many times
+        # as it rearranges the tree. This has caused problems with
+        # multivalued attributes.
+        markup = '
'
+        soup = self.soup(markup)
+        self.assertEqual(["css"], soup.div.div['class'])
+
+    def test_angle_brackets_in_attribute_values_are_escaped(self):
+        self.assertSoupEquals('', '')
+
+    def test_entities_in_attributes_converted_to_unicode(self):
+        expect = u'
'
+        self.assertSoupEquals('
', expect)
+        self.assertSoupEquals('
', expect)
+        self.assertSoupEquals('
', expect)
+        self.assertSoupEquals('
', expect)
+
+    def test_entities_in_text_converted_to_unicode(self):
+        expect = u'
pi\N{LATIN SMALL LETTER N WITH TILDE}ata
'
+        self.assertSoupEquals("
piñata
", expect)
+        self.assertSoupEquals("
piñata
", expect)
+        self.assertSoupEquals("
piñata
", expect)
+        self.assertSoupEquals("
piñata
", expect)
+
+    def test_quot_entity_converted_to_quotation_mark(self):
+        self.assertSoupEquals("
I said "good day!"
",
+                              '
I said "good day!"
')
+
+    def test_out_of_range_entity(self):
+        expect = u"\N{REPLACEMENT CHARACTER}"
+        self.assertSoupEquals("�", expect)
+        self.assertSoupEquals("�", expect)
+        self.assertSoupEquals("�", expect)
+
+    def test_multipart_strings(self):
+        "Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
+        soup = self.soup("
\nfoo
")
+        self.assertEqual("p", soup.h2.string.next_element.name)
+        self.assertEqual("p", soup.p.name)
+
+    def test_basic_namespaces(self):
+        """Parsers don't need to *understand* namespaces, but at the
+        very least they should not choke on namespaces or lose
+        data."""
+
+        markup = b'4'
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode())
+        html = soup.html
+        self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
+        self.assertEqual(
+            'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
+        self.assertEqual(
+            'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])
+
+    def test_multivalued_attribute_value_becomes_list(self):
+        markup = b''
+        soup = self.soup(markup)
+        self.assertEqual(['foo', 'bar'], soup.a['class'])
+
+    #
+    # Generally speaking, tests below this point are more tests of
+    # Beautiful Soup than tests of the tree builders. But parsers are
+    # weird, so we run these tests separately for every tree builder
+    # to detect any differences between them.
+    #
+
+    def test_can_parse_unicode_document(self):
+        # A seemingly innocuous document... but it's in Unicode! And
+        # it contains characters that can't be represented in the
+        # encoding found in the  declaration! The horror!
+        markup = u'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
+        soup = self.soup(markup)
+        self.assertEqual(u'Sacr\xe9 bleu!', soup.body.string)
+
+    def test_soupstrainer(self):
+        """Parsers should be able to work with SoupStrainers."""
+        strainer = SoupStrainer("b")
+        soup = self.soup("A bold  statement",
+                         parse_only=strainer)
+        self.assertEqual(soup.decode(), "bold")
+
+    def test_single_quote_attribute_values_become_double_quotes(self):
+        self.assertSoupEquals("",
+                              '')
+
+    def test_attribute_values_with_nested_quotes_are_left_alone(self):
+        text = """a"""
+        self.assertSoupEquals(text)
+
+    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
+        text = """a"""
+        soup = self.soup(text)
+        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
+        self.assertSoupEquals(
+            soup.foo.decode(),
+            """a""")
+
+    def test_ampersand_in_attribute_value_gets_escaped(self):
+        self.assertSoupEquals('',
+                              '')
+
+        self.assertSoupEquals(
+            'foo',
+            'foo')
+
+    def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
+        self.assertSoupEquals('')
+
+    def test_entities_in_strings_converted_during_parsing(self):
+        # Both XML and HTML entities are converted to Unicode characters
+        # during parsing.
+        text = "
<<sacré bleu!>>
"
+        expected = u"
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
"
+        self.assertSoupEquals(text, expected)
+
+    def test_smart_quotes_converted_on_the_way_in(self):
+        # Microsoft smart quotes are converted to Unicode characters during
+        # parsing.
+        quote = b"
\x91Foo\x92
"
+        soup = self.soup(quote)
+        self.assertEqual(
+            soup.p.string,
+            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
+
+    def test_non_breaking_spaces_converted_on_the_way_in(self):
+        soup = self.soup("  ")
+        self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
+
+    def test_entities_converted_on_the_way_out(self):
+        text = "
<<sacré bleu!>>
"
+        expected = u"
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
".encode("utf-8")
+        soup = self.soup(text)
+        self.assertEqual(soup.p.encode("utf-8"), expected)
+
+    def test_real_iso_latin_document(self):
+        # Smoke test of interrelated functionality, using an
+        # easy-to-understand document.
+
+        # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
+        unicode_html = u'
Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!
'
+
+        # That's because we're going to encode it into ISO-Latin-1, and use
+        # that to test.
+        iso_latin_html = unicode_html.encode("iso-8859-1")
+
+        # Parse the ISO-Latin-1 HTML.
+        soup = self.soup(iso_latin_html)
+        # Encode it to UTF-8.
+        result = soup.encode("utf-8")
+
+        # What do we expect the result to look like? Well, it would
+        # look like unicode_html, except that the META tag would say
+        # UTF-8 instead of ISO-Latin-1.
+        expected = unicode_html.replace("ISO-Latin-1", "utf-8")
+
+        # And, of course, it would be in UTF-8, not Unicode.
+        expected = expected.encode("utf-8")
+
+        # Ta-da!
+        self.assertEqual(result, expected)
+
+    def test_real_shift_jis_document(self):
+        # Smoke test to make sure the parser can handle a document in
+        # Shift-JIS encoding, without choking.
+        shift_jis_html = (
+            b'
'
+            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
+            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
+            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
+            b'
')
+        unicode_html = shift_jis_html.decode("shift-jis")
+        soup = self.soup(unicode_html)
+
+        # Make sure the parse tree is correctly encoded to various
+        # encodings.
+        self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
+        self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
+
+    def test_real_hebrew_document(self):
+        # A real-world test to make sure we can convert ISO-8859-9 (a
+        # Hebrew encoding) to UTF-8.
+        hebrew_document = b'Hebrew (ISO 8859-8) in Visual Directionality
Hebrew (ISO 8859-8) in Visual Directionality
\xed\xe5\xec\xf9'
+        soup = self.soup(
+            hebrew_document, from_encoding="iso8859-8")
+        self.assertEqual(soup.original_encoding, 'iso8859-8')
+        self.assertEqual(
+            soup.encode('utf-8'),
+            hebrew_document.decode("iso8859-8").encode("utf-8"))
+
+    def test_meta_tag_reflects_current_encoding(self):
+        # Here's the  tag saying that a document is
+        # encoded in Shift-JIS.
+        meta_tag = ('')
+
+        # Here's a document incorporating that meta tag.
+        shift_jis_html = (
+            '\n%s\n'
+            ''
+            'Shift-JIS markup goes here.') % meta_tag
+        soup = self.soup(shift_jis_html)
+
+        # Parse the document, and the charset is seemingly unaffected.
+        parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
+        content = parsed_meta['content']
+        self.assertEqual('text/html; charset=x-sjis', content)
+
+        # But that value is actually a ContentMetaAttributeValue object.
+        self.assertTrue(isinstance(content, ContentMetaAttributeValue))
+
+        # And it will take on a value that reflects its current
+        # encoding.
+        self.assertEqual('text/html; charset=utf8', content.encode("utf8"))
+
+        # For the rest of the story, see TestSubstitutions in
+        # test_tree.py.
+
+    def test_html5_style_meta_tag_reflects_current_encoding(self):
+        # Here's the  tag saying that a document is
+        # encoded in Shift-JIS.
+        meta_tag = ('')
+
+        # Here's a document incorporating that meta tag.
+        shift_jis_html = (
+            '\n%s\n'
+            ''
+            'Shift-JIS markup goes here.') % meta_tag
+        soup = self.soup(shift_jis_html)
+
+        # Parse the document, and the charset is seemingly unaffected.
+        parsed_meta = soup.find('meta', id="encoding")
+        charset = parsed_meta['charset']
+        self.assertEqual('x-sjis', charset)
+
+        # But that value is actually a CharsetMetaAttributeValue object.
+        self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))
+
+        # And it will take on a value that reflects its current
+        # encoding.
+        self.assertEqual('utf8', charset.encode("utf8"))
+
+    def test_tag_with_no_attributes_can_have_attributes_added(self):
+        data = self.soup("text")
+        data.a['foo'] = 'bar'
+        self.assertEqual('text', data.a.decode())
+
+class XMLTreeBuilderSmokeTest(object):
+
+    def test_docstring_generated(self):
+        soup = self.soup("")
+        self.assertEqual(
+            soup.encode(), b'\n')
+
+    def test_real_xhtml_document(self):
+        """A real XHTML document should come out *exactly* the same as it went in."""
+        markup = b"""
+
+
+Hello.
+Goodbye.
+"""
+        soup = self.soup(markup)
+        self.assertEqual(
+            soup.encode("utf-8"), markup)
+
+    def test_formatter_processes_script_tag_for_xml_documents(self):
+        doc = """
+  
+"""
+        soup = BeautifulSoup(doc, "xml")
+        # lxml would have stripped this while parsing, but we can add
+        # it later.
+        soup.script.string = 'console.log("< < hey > > ");'
+        encoded = soup.encode()
+        self.assertTrue(b"< < hey > >" in encoded)
+
+    def test_can_parse_unicode_document(self):
+        markup = u'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
+        soup = self.soup(markup)
+        self.assertEqual(u'Sacr\xe9 bleu!', soup.root.string)
+
+    def test_popping_namespaced_tag(self):
+        markup = 'b2012-07-02T20:33:42Zcd'
+        soup = self.soup(markup)
+        self.assertEqual(
+            unicode(soup.rss), markup)
+
+    def test_docstring_includes_correct_encoding(self):
+        soup = self.soup("")
+        self.assertEqual(
+            soup.encode("latin1"),
+            b'\n')
+
+    def test_large_xml_document(self):
+        """A large XML document should come out the same as it went in."""
+        markup = (b'\n'
+                  + b'0' * (2**12)
+                  + b'')
+        soup = self.soup(markup)
+        self.assertEqual(soup.encode("utf-8"), markup)
+
+
+    def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
+        self.assertSoupEquals("
", "
")
+        self.assertSoupEquals("
foo
")
+
+    def test_namespaces_are_preserved(self):
+        markup = 'This tag is in the a namespaceThis tag is in the b namespace'
+        soup = self.soup(markup)
+        root = soup.root
+        self.assertEqual("http://example.com/", root['xmlns:a'])
+        self.assertEqual("http://example.net/", root['xmlns:b'])
+
+    def test_closing_namespaced_tag(self):
+        markup = '
20010504
'
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.p), markup)
+
+    def test_namespaced_attributes(self):
+        markup = ''
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.foo), markup)
+
+    def test_namespaced_attributes_xml_namespace(self):
+        markup = 'bar'
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.foo), markup)
+
+class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
+    """Smoke test for a tree builder that supports HTML5."""
+
+    def test_real_xhtml_document(self):
+        # Since XHTML is not HTML5, HTML5 parsers are not tested to handle
+        # XHTML documents in any particular way.
+        pass
+
+    def test_html_tags_have_namespace(self):
+        markup = ""
+        soup = self.soup(markup)
+        self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)
+
+    def test_svg_tags_have_namespace(self):
+        markup = ''
+        soup = self.soup(markup)
+        namespace = "http://www.w3.org/2000/svg"
+        self.assertEqual(namespace, soup.svg.namespace)
+        self.assertEqual(namespace, soup.circle.namespace)
+
+
+    def test_mathml_tags_have_namespace(self):
+        markup = '5'
+        soup = self.soup(markup)
+        namespace = 'http://www.w3.org/1998/Math/MathML'
+        self.assertEqual(namespace, soup.math.namespace)
+        self.assertEqual(namespace, soup.msqrt.namespace)
+
+    def test_xml_declaration_becomes_comment(self):
+        markup = ''
+        soup = self.soup(markup)
+        self.assertTrue(isinstance(soup.contents[0], Comment))
+        self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?')
+        self.assertEqual("html", soup.contents[0].next_element.name)
+
+def skipIf(condition, reason):
+   def nothing(test, *args, **kwargs):
+       return None
+
+   def decorator(test_item):
+       if condition:
+           return nothing
+       else:
+           return test_item
+
+   return decorator
diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/tests/__init__.py b/import-layers/yocto-poky/bitbake/lib/bs4/tests/__init__.py
new file mode 100644
index 000000000..142c8cc3f
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/bs4/tests/__init__.py
@@ -0,0 +1 @@
+"The beautifulsoup tests."
diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_builder_registry.py b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_builder_registry.py
new file mode 100644
index 000000000..92ad10fb0
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_builder_registry.py
@@ -0,0 +1,141 @@
+"""Tests of the builder registry."""
+
+import unittest
+
+from bs4 import BeautifulSoup
+from bs4.builder import (
+    builder_registry as registry,
+    HTMLParserTreeBuilder,
+    TreeBuilderRegistry,
+)
+
+try:
+    from bs4.builder import HTML5TreeBuilder
+    HTML5LIB_PRESENT = True
+except ImportError:
+    HTML5LIB_PRESENT = False
+
+try:
+    from bs4.builder import (
+        LXMLTreeBuilderForXML,
+        LXMLTreeBuilder,
+        )
+    LXML_PRESENT = True
+except ImportError:
+    LXML_PRESENT = False
+
+
+class BuiltInRegistryTest(unittest.TestCase):
+    """Test the built-in registry with the default builders registered."""
+
+    def test_combination(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('fast', 'html'),
+                             LXMLTreeBuilder)
+
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('permissive', 'xml'),
+                             LXMLTreeBuilderForXML)
+        self.assertEqual(registry.lookup('strict', 'html'),
+                          HTMLParserTreeBuilder)
+        if HTML5LIB_PRESENT:
+            self.assertEqual(registry.lookup('html5lib', 'html'),
+                              HTML5TreeBuilder)
+
+    def test_lookup_by_markup_type(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('html'), LXMLTreeBuilder)
+            self.assertEqual(registry.lookup('xml'), LXMLTreeBuilderForXML)
+        else:
+            self.assertEqual(registry.lookup('xml'), None)
+            if HTML5LIB_PRESENT:
+                self.assertEqual(registry.lookup('html'), HTML5TreeBuilder)
+            else:
+                self.assertEqual(registry.lookup('html'), HTMLParserTreeBuilder)
+
+    def test_named_library(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('lxml', 'xml'),
+                             LXMLTreeBuilderForXML)
+            self.assertEqual(registry.lookup('lxml', 'html'),
+                             LXMLTreeBuilder)
+        if HTML5LIB_PRESENT:
+            self.assertEqual(registry.lookup('html5lib'),
+                              HTML5TreeBuilder)
+
+        self.assertEqual(registry.lookup('html.parser'),
+                          HTMLParserTreeBuilder)
+
+    def test_beautifulsoup_constructor_does_lookup(self):
+        # You can pass in a string.
+        BeautifulSoup("", features="html")
+        # Or a list of strings.
+        BeautifulSoup("", features=["html", "fast"])
+
+        # You'll get an exception if BS can't find an appropriate
+        # builder.
+        self.assertRaises(ValueError, BeautifulSoup,
+                          "", features="no-such-feature")
+
+class RegistryTest(unittest.TestCase):
+    """Test the TreeBuilderRegistry class in general."""
+
+    def setUp(self):
+        self.registry = TreeBuilderRegistry()
+
+    def builder_for_features(self, *feature_list):
+        cls = type('Builder_' + '_'.join(feature_list),
+                   (object,), {'features' : feature_list})
+
+        self.registry.register(cls)
+        return cls
+
+    def test_register_with_no_features(self):
+        builder = self.builder_for_features()
+
+        # Since the builder advertises no features, you can't find it
+        # by looking up features.
+        self.assertEqual(self.registry.lookup('foo'), None)
+
+        # But you can find it by doing a lookup with no features, if
+        # this happens to be the only registered builder.
+        self.assertEqual(self.registry.lookup(), builder)
+
+    def test_register_with_features_makes_lookup_succeed(self):
+        builder = self.builder_for_features('foo', 'bar')
+        self.assertEqual(self.registry.lookup('foo'), builder)
+        self.assertEqual(self.registry.lookup('bar'), builder)
+
+    def test_lookup_fails_when_no_builder_implements_feature(self):
+        builder = self.builder_for_features('foo', 'bar')
+        self.assertEqual(self.registry.lookup('baz'), None)
+
+    def test_lookup_gets_most_recent_registration_when_no_feature_specified(self):
+        builder1 = self.builder_for_features('foo')
+        builder2 = self.builder_for_features('bar')
+        self.assertEqual(self.registry.lookup(), builder2)
+
+    def test_lookup_fails_when_no_tree_builders_registered(self):
+        self.assertEqual(self.registry.lookup(), None)
+
+    def test_lookup_gets_most_recent_builder_supporting_all_features(self):
+        has_one = self.builder_for_features('foo')
+        has_the_other = self.builder_for_features('bar')
+        has_both_early = self.builder_for_features('foo', 'bar', 'baz')
+        has_both_late = self.builder_for_features('foo', 'bar', 'quux')
+        lacks_one = self.builder_for_features('bar')
+        has_the_other = self.builder_for_features('foo')
+
+        # There are two builders featuring 'foo' and 'bar', but
+        # the one that also features 'quux' was registered later.
+        self.assertEqual(self.registry.lookup('foo', 'bar'),
+                          has_both_late)
+
+        # There is only one builder featuring 'foo', 'bar', and 'baz'.
+        self.assertEqual(self.registry.lookup('foo', 'bar', 'baz'),
+                          has_both_early)
+
+    def test_lookup_fails_when_cannot_reconcile_requested_features(self):
+        builder1 = self.builder_for_features('foo', 'bar')
+        builder2 = self.builder_for_features('foo', 'baz')
+        self.assertEqual(self.registry.lookup('bar', 'baz'), None)
diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_docs.py b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_docs.py
new file mode 100644
index 000000000..5b9f67709
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_docs.py
@@ -0,0 +1,36 @@
+"Test harness for doctests."
+
+# pylint: disable-msg=E0611,W0142
+
+__metaclass__ = type
+__all__ = [
+    'additional_tests',
+    ]
+
+import atexit
+import doctest
+import os
+#from pkg_resources import (
+#    resource_filename, resource_exists, resource_listdir, cleanup_resources)
+import unittest
+
+DOCTEST_FLAGS = (
+    doctest.ELLIPSIS |
+    doctest.NORMALIZE_WHITESPACE |
+    doctest.REPORT_NDIFF)
+
+
+# def additional_tests():
+#     "Run the doc tests (README.txt and docs/*, if any exist)"
+#     doctest_files = [
+#         os.path.abspath(resource_filename('bs4', 'README.txt'))]
+#     if resource_exists('bs4', 'docs'):
+#         for name in resource_listdir('bs4', 'docs'):
+#             if name.endswith('.txt'):
+#                 doctest_files.append(
+#                     os.path.abspath(
+#                         resource_filename('bs4', 'docs/%s' % name)))
+#     kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS)
+#     atexit.register(cleanup_resources)
+#     return unittest.TestSuite((
+#         doctest.DocFileSuite(*doctest_files, **kwargs)))
diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_html5lib.py b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_html5lib.py
new file mode 100644
index 000000000..594c3e1f2
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_html5lib.py
@@ -0,0 +1,85 @@
+"""Tests to ensure that the html5lib tree builder generates good trees."""
+
+import warnings
+
+try:
+    from bs4.builder import HTML5TreeBuilder
+    HTML5LIB_PRESENT = True
+except ImportError, e:
+    HTML5LIB_PRESENT = False
+from bs4.element import SoupStrainer
+from bs4.testing import (
+    HTML5TreeBuilderSmokeTest,
+    SoupTest,
+    skipIf,
+)
+
+@skipIf(
+    not HTML5LIB_PRESENT,
+    "html5lib seems not to be present, not testing its tree builder.")
+class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest):
+    """See ``HTML5TreeBuilderSmokeTest``."""
+
+    @property
+    def default_builder(self):
+        return HTML5TreeBuilder()
+
+    def test_soupstrainer(self):
+        # The html5lib tree builder does not support SoupStrainers.
+        strainer = SoupStrainer("b")
+        markup = "
A bold statement.
"
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup(markup, parse_only=strainer)
+        self.assertEqual(
+            soup.decode(), self.document_for(markup))
+
+        self.assertTrue(
+            "the html5lib tree builder doesn't support parse_only" in
+            str(w[0].message))
+
+    def test_correctly_nested_tables(self):
+        """html5lib inserts  tags where other parsers don't."""
+        markup = (''
+                  ''
+                  "')
+
+        self.assertSoupEquals(
+            markup,
+            '
Here's another table:"
+                  ''
+                  ''
+                  '
foo
Here\'s another table:'
+            '
foo
'
+            '
')
+
+        self.assertSoupEquals(
+            ""
+            ""
+            "
Foo
Bar
Baz
")
+
+    def test_xml_declaration_followed_by_doctype(self):
+        markup = '''
+
+
+  
+  
+  
+   
foo

+  
+'''
+        soup = self.soup(markup)
+        # Verify that we can reach the 
 tag; this means the tree is connected.
+        self.assertEqual(b"
foo
", soup.p.encode())
+
+    def test_reparented_markup(self):
+        markup = '
foo
\n
bar
'
+        soup = self.soup(markup)
+        self.assertEqual(u"
foo
\n
bar
", soup.body.decode())
+        self.assertEqual(2, len(soup.find_all('p')))
+
+
+    def test_reparented_markup_ends_with_whitespace(self):
+        markup = '
foo
\n
bar
\n'
+        soup = self.soup(markup)
+        self.assertEqual(u"
foo
\n
bar
\n", soup.body.decode())
+        self.assertEqual(2, len(soup.find_all('p')))
diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_htmlparser.py b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_htmlparser.py
new file mode 100644
index 000000000..bcb5ed232
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_htmlparser.py
@@ -0,0 +1,19 @@
+"""Tests to ensure that the html.parser tree builder generates good
+trees."""
+
+from bs4.testing import SoupTest, HTMLTreeBuilderSmokeTest
+from bs4.builder import HTMLParserTreeBuilder
+
+class HTMLParserTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
+
+    @property
+    def default_builder(self):
+        return HTMLParserTreeBuilder()
+
+    def test_namespaced_system_doctype(self):
+        # html.parser can't handle namespaced doctypes, so skip this one.
+        pass
+
+    def test_namespaced_public_doctype(self):
+        # html.parser can't handle namespaced doctypes, so skip this one.
+        pass
diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_lxml.py b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_lxml.py
new file mode 100644
index 000000000..2b2e9b7e7
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_lxml.py
@@ -0,0 +1,91 @@
+"""Tests to ensure that the lxml tree builder generates good trees."""
+
+import re
+import warnings
+
+try:
+    import lxml.etree
+    LXML_PRESENT = True
+    LXML_VERSION = lxml.etree.LXML_VERSION
+except ImportError, e:
+    LXML_PRESENT = False
+    LXML_VERSION = (0,)
+
+if LXML_PRESENT:
+    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
+
+from bs4 import (
+    BeautifulSoup,
+    BeautifulStoneSoup,
+    )
+from bs4.element import Comment, Doctype, SoupStrainer
+from bs4.testing import skipIf
+from bs4.tests import test_htmlparser
+from bs4.testing import (
+    HTMLTreeBuilderSmokeTest,
+    XMLTreeBuilderSmokeTest,
+    SoupTest,
+    skipIf,
+)
+
+@skipIf(
+    not LXML_PRESENT,
+    "lxml seems not to be present, not testing its tree builder.")
+class LXMLTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
+    """See ``HTMLTreeBuilderSmokeTest``."""
+
+    @property
+    def default_builder(self):
+        return LXMLTreeBuilder()
+
+    def test_out_of_range_entity(self):
+        self.assertSoupEquals(
+            "
foo�bar
", "
foobar
")
+        self.assertSoupEquals(
+            "
foo�bar
", "
foobar
")
+        self.assertSoupEquals(
+            "
foo�bar
", "
foobar
")
+
+    # In lxml < 2.3.5, an empty doctype causes a segfault. Skip this
+    # test if an old version of lxml is installed.
+
+    @skipIf(
+        not LXML_PRESENT or LXML_VERSION < (2,3,5,0),
+        "Skipping doctype test for old version of lxml to avoid segfault.")
+    def test_empty_doctype(self):
+        soup = self.soup("")
+        doctype = soup.contents[0]
+        self.assertEqual("", doctype.strip())
+
+    def test_beautifulstonesoup_is_xml_parser(self):
+        # Make sure that the deprecated BSS class uses an xml builder
+        # if one is installed.
+        with warnings.catch_warnings(record=True) as w:
+            soup = BeautifulStoneSoup("")
+        self.assertEqual(u"", unicode(soup.b))
+        self.assertTrue("BeautifulStoneSoup class is deprecated" in str(w[0].message))
+
+    def test_real_xhtml_document(self):
+        """lxml strips the XML definition from an XHTML doc, which is fine."""
+        markup = b"""
+
+
+Hello.
+Goodbye.
+"""
+        soup = self.soup(markup)
+        self.assertEqual(
+            soup.encode("utf-8").replace(b"\n", b''),
+            markup.replace(b'\n', b'').replace(
+                b'', b''))
+
+
+@skipIf(
+    not LXML_PRESENT,
+    "lxml seems not to be present, not testing its XML tree builder.")
+class LXMLXMLTreeBuilderSmokeTest(SoupTest, XMLTreeBuilderSmokeTest):
+    """See ``HTMLTreeBuilderSmokeTest``."""
+
+    @property
+    def default_builder(self):
+        return LXMLTreeBuilderForXML()
diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_soup.py b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_soup.py
new file mode 100644
index 000000000..47ac245f9
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_soup.py
@@ -0,0 +1,434 @@
+# -*- coding: utf-8 -*-
+"""Tests of Beautiful Soup as a whole."""
+
+import logging
+import unittest
+import sys
+import tempfile
+
+from bs4 import (
+    BeautifulSoup,
+    BeautifulStoneSoup,
+)
+from bs4.element import (
+    CharsetMetaAttributeValue,
+    ContentMetaAttributeValue,
+    SoupStrainer,
+    NamespacedAttribute,
+    )
+import bs4.dammit
+from bs4.dammit import (
+    EntitySubstitution,
+    UnicodeDammit,
+)
+from bs4.testing import (
+    SoupTest,
+    skipIf,
+)
+import warnings
+
+try:
+    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
+    LXML_PRESENT = True
+except ImportError, e:
+    LXML_PRESENT = False
+
+PYTHON_2_PRE_2_7 = (sys.version_info < (2,7))
+PYTHON_3_PRE_3_2 = (sys.version_info[0] == 3 and sys.version_info < (3,2))
+
+class TestConstructor(SoupTest):
+
+    def test_short_unicode_input(self):
+        data = u"
éé
"
+        soup = self.soup(data)
+        self.assertEqual(u"éé", soup.h1.string)
+
+    def test_embedded_null(self):
+        data = u"
foo\0bar
"
+        soup = self.soup(data)
+        self.assertEqual(u"foo\0bar", soup.h1.string)
+
+
+class TestDeprecatedConstructorArguments(SoupTest):
+
+    def test_parseOnlyThese_renamed_to_parse_only(self):
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup("", parseOnlyThese=SoupStrainer("b"))
+        msg = str(w[0].message)
+        self.assertTrue("parseOnlyThese" in msg)
+        self.assertTrue("parse_only" in msg)
+        self.assertEqual(b"", soup.encode())
+
+    def test_fromEncoding_renamed_to_from_encoding(self):
+        with warnings.catch_warnings(record=True) as w:
+            utf8 = b"\xc3\xa9"
+            soup = self.soup(utf8, fromEncoding="utf8")
+        msg = str(w[0].message)
+        self.assertTrue("fromEncoding" in msg)
+        self.assertTrue("from_encoding" in msg)
+        self.assertEqual("utf8", soup.original_encoding)
+
+    def test_unrecognized_keyword_argument(self):
+        self.assertRaises(
+            TypeError, self.soup, "", no_such_argument=True)
+
+class TestWarnings(SoupTest):
+
+    def test_disk_file_warning(self):
+        filehandle = tempfile.NamedTemporaryFile()
+        filename = filehandle.name
+        try:
+            with warnings.catch_warnings(record=True) as w:
+                soup = self.soup(filename)
+            msg = str(w[0].message)
+            self.assertTrue("looks like a filename" in msg)
+        finally:
+            filehandle.close()
+
+        # The file no longer exists, so Beautiful Soup will no longer issue the warning.
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup(filename)
+        self.assertEqual(0, len(w))
+
+    def test_url_warning(self):
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup("http://www.crummy.com/")
+        msg = str(w[0].message)
+        self.assertTrue("looks like a URL" in msg)
+
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup("http://www.crummy.com/ is great")
+        self.assertEqual(0, len(w))
+
+class TestSelectiveParsing(SoupTest):
+
+    def test_parse_with_soupstrainer(self):
+        markup = "NoYesNoYes Yes"
+        strainer = SoupStrainer("b")
+        soup = self.soup(markup, parse_only=strainer)
+        self.assertEqual(soup.encode(), b"YesYes Yes")
+
+
+class TestEntitySubstitution(unittest.TestCase):
+    """Standalone tests of the EntitySubstitution class."""
+    def setUp(self):
+        self.sub = EntitySubstitution
+
+    def test_simple_html_substitution(self):
+        # Unicode characters corresponding to named HTML entites
+        # are substituted, and no others.
+        s = u"foo\u2200\N{SNOWMAN}\u00f5bar"
+        self.assertEqual(self.sub.substitute_html(s),
+                          u"foo∀\N{SNOWMAN}õbar")
+
+    def test_smart_quote_substitution(self):
+        # MS smart quotes are a common source of frustration, so we
+        # give them a special test.
+        quotes = b"\x91\x92foo\x93\x94"
+        dammit = UnicodeDammit(quotes)
+        self.assertEqual(self.sub.substitute_html(dammit.markup),
+                          "‘’foo“”")
+
+    def test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false(self):
+        s = 'Welcome to "my bar"'
+        self.assertEqual(self.sub.substitute_xml(s, False), s)
+
+    def test_xml_attribute_quoting_normally_uses_double_quotes(self):
+        self.assertEqual(self.sub.substitute_xml("Welcome", True),
+                          '"Welcome"')
+        self.assertEqual(self.sub.substitute_xml("Bob's Bar", True),
+                          '"Bob\'s Bar"')
+
+    def test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes(self):
+        s = 'Welcome to "my bar"'
+        self.assertEqual(self.sub.substitute_xml(s, True),
+                          "'Welcome to \"my bar\"'")
+
+    def test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes(self):
+        s = 'Welcome to "Bob\'s Bar"'
+        self.assertEqual(
+            self.sub.substitute_xml(s, True),
+            '"Welcome to "Bob\'s Bar""')
+
+    def test_xml_quotes_arent_escaped_when_value_is_not_being_quoted(self):
+        quoted = 'Welcome to "Bob\'s Bar"'
+        self.assertEqual(self.sub.substitute_xml(quoted), quoted)
+
+    def test_xml_quoting_handles_angle_brackets(self):
+        self.assertEqual(
+            self.sub.substitute_xml("foo"),
+            "foo<bar>")
+
+    def test_xml_quoting_handles_ampersands(self):
+        self.assertEqual(self.sub.substitute_xml("AT&T"), "AT&T")
+
+    def test_xml_quoting_including_ampersands_when_they_are_part_of_an_entity(self):
+        self.assertEqual(
+            self.sub.substitute_xml("ÁT&T"),
+            "&Aacute;T&T")
+
+    def test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entity(self):
+        self.assertEqual(
+            self.sub.substitute_xml_containing_entities("ÁT&T"),
+            "ÁT&T")
+
+    def test_quotes_not_html_substituted(self):
+        """There's no need to do this except inside attribute values."""
+        text = 'Bob\'s "bar"'
+        self.assertEqual(self.sub.substitute_html(text), text)
+
+
+class TestEncodingConversion(SoupTest):
+    # Test Beautiful Soup's ability to decode and encode from various
+    # encodings.
+
+    def setUp(self):
+        super(TestEncodingConversion, self).setUp()
+        self.unicode_data = u'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
+        self.utf8_data = self.unicode_data.encode("utf-8")
+        # Just so you know what it looks like.
+        self.assertEqual(
+            self.utf8_data,
+            b'Sacr\xc3\xa9 bleu!')
+
+    def test_ascii_in_unicode_out(self):
+        # ASCII input is converted to Unicode. The original_encoding
+        # attribute is set to 'utf-8', a superset of ASCII.
+        chardet = bs4.dammit.chardet_dammit
+        logging.disable(logging.WARNING)
+        try:
+            def noop(str):
+                return None
+            # Disable chardet, which will realize that the ASCII is ASCII.
+            bs4.dammit.chardet_dammit = noop
+            ascii = b"a"
+            soup_from_ascii = self.soup(ascii)
+            unicode_output = soup_from_ascii.decode()
+            self.assertTrue(isinstance(unicode_output, unicode))
+            self.assertEqual(unicode_output, self.document_for(ascii.decode()))
+            self.assertEqual(soup_from_ascii.original_encoding.lower(), "utf-8")
+        finally:
+            logging.disable(logging.NOTSET)
+            bs4.dammit.chardet_dammit = chardet
+
+    def test_unicode_in_unicode_out(self):
+        # Unicode input is left alone. The original_encoding attribute
+        # is not set.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEqual(soup_from_unicode.decode(), self.unicode_data)
+        self.assertEqual(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!')
+        self.assertEqual(soup_from_unicode.original_encoding, None)
+
+    def test_utf8_in_unicode_out(self):
+        # UTF-8 input is converted to Unicode. The original_encoding
+        # attribute is set.
+        soup_from_utf8 = self.soup(self.utf8_data)
+        self.assertEqual(soup_from_utf8.decode(), self.unicode_data)
+        self.assertEqual(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!')
+
+    def test_utf8_out(self):
+        # The internal data structures can be encoded as UTF-8.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEqual(soup_from_unicode.encode('utf-8'), self.utf8_data)
+
+    @skipIf(
+        PYTHON_2_PRE_2_7 or PYTHON_3_PRE_3_2,
+        "Bad HTMLParser detected; skipping test of non-ASCII characters in attribute name.")
+    def test_attribute_name_containing_unicode_characters(self):
+        markup = u'
'
+        self.assertEqual(self.soup(markup).div.encode("utf8"), markup.encode("utf8"))
+
+class TestUnicodeDammit(unittest.TestCase):
+    """Standalone tests of UnicodeDammit."""
+
+    def test_unicode_input(self):
+        markup = u"I'm already Unicode! \N{SNOWMAN}"
+        dammit = UnicodeDammit(markup)
+        self.assertEqual(dammit.unicode_markup, markup)
+
+    def test_smart_quotes_to_unicode(self):
+        markup = b"\x91\x92\x93\x94"
+        dammit = UnicodeDammit(markup)
+        self.assertEqual(
+            dammit.unicode_markup, u"\u2018\u2019\u201c\u201d")
+
+    def test_smart_quotes_to_xml_entities(self):
+        markup = b"\x91\x92\x93\x94"
+        dammit = UnicodeDammit(markup, smart_quotes_to="xml")
+        self.assertEqual(
+            dammit.unicode_markup, "‘’“”")
+
+    def test_smart_quotes_to_html_entities(self):
+        markup = b"\x91\x92\x93\x94"
+        dammit = UnicodeDammit(markup, smart_quotes_to="html")
+        self.assertEqual(
+            dammit.unicode_markup, "‘’“”")
+
+    def test_smart_quotes_to_ascii(self):
+        markup = b"\x91\x92\x93\x94"
+        dammit = UnicodeDammit(markup, smart_quotes_to="ascii")
+        self.assertEqual(
+            dammit.unicode_markup, """''""""")
+
+    def test_detect_utf8(self):
+        utf8 = b"\xc3\xa9"
+        dammit = UnicodeDammit(utf8)
+        self.assertEqual(dammit.unicode_markup, u'\xe9')
+        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+
+    def test_convert_hebrew(self):
+        hebrew = b"\xed\xe5\xec\xf9"
+        dammit = UnicodeDammit(hebrew, ["iso-8859-8"])
+        self.assertEqual(dammit.original_encoding.lower(), 'iso-8859-8')
+        self.assertEqual(dammit.unicode_markup, u'\u05dd\u05d5\u05dc\u05e9')
+
+    def test_dont_see_smart_quotes_where_there_are_none(self):
+        utf_8 = b"\343\202\261\343\203\274\343\202\277\343\202\244 Watch"
+        dammit = UnicodeDammit(utf_8)
+        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+        self.assertEqual(dammit.unicode_markup.encode("utf-8"), utf_8)
+
+    def test_ignore_inappropriate_codecs(self):
+        utf8_data = u"Räksmörgås".encode("utf-8")
+        dammit = UnicodeDammit(utf8_data, ["iso-8859-8"])
+        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+
+    def test_ignore_invalid_codecs(self):
+        utf8_data = u"Räksmörgås".encode("utf-8")
+        for bad_encoding in ['.utf8', '...', 'utF---16.!']:
+            dammit = UnicodeDammit(utf8_data, [bad_encoding])
+            self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+
+    def test_detect_html5_style_meta_tag(self):
+
+        for data in (
+            b'',
+            b"",
+            b"",
+            b""):
+            dammit = UnicodeDammit(data, is_html=True)
+            self.assertEqual(
+                "euc-jp", dammit.original_encoding)
+
+    def test_last_ditch_entity_replacement(self):
+        # This is a UTF-8 document that contains bytestrings
+        # completely incompatible with UTF-8 (ie. encoded with some other
+        # encoding).
+        #
+        # Since there is no consistent encoding for the document,
+        # Unicode, Dammit will eventually encode the document as UTF-8
+        # and encode the incompatible characters as REPLACEMENT
+        # CHARACTER.
+        #
+        # If chardet is installed, it will detect that the document
+        # can be converted into ISO-8859-1 without errors. This happens
+        # to be the wrong encoding, but it is a consistent encoding, so the
+        # code we're testing here won't run.
+        #
+        # So we temporarily disable chardet if it's present.
+        doc = b"""\357\273\277
+\330\250\330\252\330\261
+\310\322\321\220\312\321\355\344"""
+        chardet = bs4.dammit.chardet_dammit
+        logging.disable(logging.WARNING)
+        try:
+            def noop(str):
+                return None
+            bs4.dammit.chardet_dammit = noop
+            dammit = UnicodeDammit(doc)
+            self.assertEqual(True, dammit.contains_replacement_characters)
+            self.assertTrue(u"\ufffd" in dammit.unicode_markup)
+
+            soup = BeautifulSoup(doc, "html.parser")
+            self.assertTrue(soup.contains_replacement_characters)
+        finally:
+            logging.disable(logging.NOTSET)
+            bs4.dammit.chardet_dammit = chardet
+
+    def test_byte_order_mark_removed(self):
+        # A document written in UTF-16LE will have its byte order marker stripped.
+        data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00'
+        dammit = UnicodeDammit(data)
+        self.assertEqual(u"áé", dammit.unicode_markup)
+        self.assertEqual("utf-16le", dammit.original_encoding)
+
+    def test_detwingle(self):
+        # Here's a UTF8 document.
+        utf8 = (u"\N{SNOWMAN}" * 3).encode("utf8")
+
+        # Here's a Windows-1252 document.
+        windows_1252 = (
+            u"\N{LEFT DOUBLE QUOTATION MARK}Hi, I like Windows!"
+            u"\N{RIGHT DOUBLE QUOTATION MARK}").encode("windows_1252")
+
+        # Through some unholy alchemy, they've been stuck together.
+        doc = utf8 + windows_1252 + utf8
+
+        # The document can't be turned into UTF-8:
+        self.assertRaises(UnicodeDecodeError, doc.decode, "utf8")
+
+        # Unicode, Dammit thinks the whole document is Windows-1252,
+        # and decodes it into "☃☃☃“Hi, I like Windows!â€Ã¢ËœÆ’☃☃"
+
+        # But if we run it through fix_embedded_windows_1252, it's fixed:
+
+        fixed = UnicodeDammit.detwingle(doc)
+        self.assertEqual(
+            u"☃☃☃“Hi, I like Windows!â€â˜ƒâ˜ƒâ˜ƒ", fixed.decode("utf8"))
+
+    def test_detwingle_ignores_multibyte_characters(self):
+        # Each of these characters has a UTF-8 representation ending
+        # in \x93. \x93 is a smart quote if interpreted as
+        # Windows-1252. But our code knows to skip over multibyte
+        # UTF-8 characters, so they'll survive the process unscathed.
+        for tricky_unicode_char in (
+            u"\N{LATIN SMALL LIGATURE OE}", # 2-byte char '\xc5\x93'
+            u"\N{LATIN SUBSCRIPT SMALL LETTER X}", # 3-byte char '\xe2\x82\x93'
+            u"\xf0\x90\x90\x93", # This is a CJK character, not sure which one.
+            ):
+            input = tricky_unicode_char.encode("utf8")
+            self.assertTrue(input.endswith(b'\x93'))
+            output = UnicodeDammit.detwingle(input)
+            self.assertEqual(output, input)
+
+class TestNamedspacedAttribute(SoupTest):
+
+    def test_name_may_be_none(self):
+        a = NamespacedAttribute("xmlns", None)
+        self.assertEqual(a, "xmlns")
+
+    def test_attribute_is_equivalent_to_colon_separated_string(self):
+        a = NamespacedAttribute("a", "b")
+        self.assertEqual("a:b", a)
+
+    def test_attributes_are_equivalent_if_prefix_and_name_identical(self):
+        a = NamespacedAttribute("a", "b", "c")
+        b = NamespacedAttribute("a", "b", "c")
+        self.assertEqual(a, b)
+
+        # The actual namespace is not considered.
+        c = NamespacedAttribute("a", "b", None)
+        self.assertEqual(a, c)
+
+        # But name and prefix are important.
+        d = NamespacedAttribute("a", "z", "c")
+        self.assertNotEqual(a, d)
+
+        e = NamespacedAttribute("z", "b", "c")
+        self.assertNotEqual(a, e)
+
+
+class TestAttributeValueWithCharsetSubstitution(unittest.TestCase):
+
+    def test_content_meta_attribute_value(self):
+        value = CharsetMetaAttributeValue("euc-jp")
+        self.assertEqual("euc-jp", value)
+        self.assertEqual("euc-jp", value.original_value)
+        self.assertEqual("utf8", value.encode("utf8"))
+
+
+    def test_content_meta_attribute_value(self):
+        value = ContentMetaAttributeValue("text/html; charset=euc-jp")
+        self.assertEqual("text/html; charset=euc-jp", value)
+        self.assertEqual("text/html; charset=euc-jp", value.original_value)
+        self.assertEqual("text/html; charset=utf8", value.encode("utf8"))
diff --git a/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_tree.py b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_tree.py
new file mode 100644
index 000000000..f8515c0ea
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/bs4/tests/test_tree.py
@@ -0,0 +1,1829 @@
+# -*- coding: utf-8 -*-
+"""Tests for Beautiful Soup's tree traversal methods.
+
+The tree traversal methods are the main advantage of using Beautiful
+Soup over just using a parser.
+
+Different parsers will build different Beautiful Soup trees given the
+same markup, but all Beautiful Soup trees can be traversed with the
+methods tested here.
+"""
+
+import copy
+import pickle
+import re
+import warnings
+from bs4 import BeautifulSoup
+from bs4.builder import (
+    builder_registry,
+    HTMLParserTreeBuilder,
+)
+from bs4.element import (
+    CData,
+    Comment,
+    Doctype,
+    NavigableString,
+    SoupStrainer,
+    Tag,
+)
+from bs4.testing import (
+    SoupTest,
+    skipIf,
+)
+
+XML_BUILDER_PRESENT = (builder_registry.lookup("xml") is not None)
+LXML_PRESENT = (builder_registry.lookup("lxml") is not None)
+
+class TreeTest(SoupTest):
+
+    def assertSelects(self, tags, should_match):
+        """Make sure that the given tags have the correct text.
+
+        This is used in tests that define a bunch of tags, each
+        containing a single string, and then select certain strings by
+        some mechanism.
+        """
+        self.assertEqual([tag.string for tag in tags], should_match)
+
+    def assertSelectsIDs(self, tags, should_match):
+        """Make sure that the given tags have the correct IDs.
+
+        This is used in tests that define a bunch of tags, each
+        containing a single string, and then select certain strings by
+        some mechanism.
+        """
+        self.assertEqual([tag['id'] for tag in tags], should_match)
+
+
+class TestFind(TreeTest):
+    """Basic tests of the find() method.
+
+    find() just calls find_all() with limit=1, so it's not tested all
+    that thouroughly here.
+    """
+
+    def test_find_tag(self):
+        soup = self.soup("1234")
+        self.assertEqual(soup.find("b").string, "2")
+
+    def test_unicode_text_find(self):
+        soup = self.soup(u'
Räksmörgås
')
+        self.assertEqual(soup.find(text=u'Räksmörgås'), u'Räksmörgås')
+
+    def test_find_everything(self):
+        """Test an optimization that finds all tags."""
+        soup = self.soup("foobar")
+        self.assertEqual(2, len(soup.find_all()))
+
+    def test_find_everything_with_name(self):
+        """Test an optimization that finds all tags with a given name."""
+        soup = self.soup("foobarbaz")
+        self.assertEqual(2, len(soup.find_all('a')))
+
+class TestFindAll(TreeTest):
+    """Basic tests of the find_all() method."""
+
+    def test_find_all_text_nodes(self):
+        """You can search the tree for text nodes."""
+        soup = self.soup("Foobar\xbb")
+        # Exact match.
+        self.assertEqual(soup.find_all(text="bar"), [u"bar"])
+        # Match any of a number of strings.
+        self.assertEqual(
+            soup.find_all(text=["Foo", "bar"]), [u"Foo", u"bar"])
+        # Match a regular expression.
+        self.assertEqual(soup.find_all(text=re.compile('.*')),
+                         [u"Foo", u"bar", u'\xbb'])
+        # Match anything.
+        self.assertEqual(soup.find_all(text=True),
+                         [u"Foo", u"bar", u'\xbb'])
+
+    def test_find_all_limit(self):
+        """You can limit the number of items returned by find_all."""
+        soup = self.soup("12345")
+        self.assertSelects(soup.find_all('a', limit=3), ["1", "2", "3"])
+        self.assertSelects(soup.find_all('a', limit=1), ["1"])
+        self.assertSelects(
+            soup.find_all('a', limit=10), ["1", "2", "3", "4", "5"])
+
+        # A limit of 0 means no limit.
+        self.assertSelects(
+            soup.find_all('a', limit=0), ["1", "2", "3", "4", "5"])
+
+    def test_calling_a_tag_is_calling_findall(self):
+        soup = self.soup("123")
+        self.assertSelects(soup('a', limit=1), ["1"])
+        self.assertSelects(soup.b(id="foo"), ["3"])
+
+    def test_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursion(self):
+        soup = self.soup("")
+        # Create a self-referential list.
+        l = []
+        l.append(l)
+
+        # Without special code in _normalize_search_value, this would cause infinite
+        # recursion.
+        self.assertEqual([], soup.find_all(l))
+
+    def test_find_all_resultset(self):
+        """All find_all calls return a ResultSet"""
+        soup = self.soup("")
+        result = soup.find_all("a")
+        self.assertTrue(hasattr(result, "source"))
+
+        result = soup.find_all(True)
+        self.assertTrue(hasattr(result, "source"))
+
+        result = soup.find_all(text="foo")
+        self.assertTrue(hasattr(result, "source"))
+
+
+class TestFindAllBasicNamespaces(TreeTest):
+
+    def test_find_by_namespaced_name(self):
+        soup = self.soup('4')
+        self.assertEqual("4", soup.find("mathml:msqrt").string)
+        self.assertEqual("a", soup.find(attrs= { "svg:fill" : "red" }).name)
+
+
+class TestFindAllByName(TreeTest):
+    """Test ways of finding tags by tag name."""
+
+    def setUp(self):
+        super(TreeTest, self).setUp()
+        self.tree =  self.soup("""First tag.
+                                  Second tag.
+                                  Third Nested tag. tag.""")
+
+    def test_find_all_by_tag_name(self):
+        # Find all the  tags.
+        self.assertSelects(
+            self.tree.find_all('a'), ['First tag.', 'Nested tag.'])
+
+    def test_find_all_by_name_and_text(self):
+        self.assertSelects(
+            self.tree.find_all('a', text='First tag.'), ['First tag.'])
+
+        self.assertSelects(
+            self.tree.find_all('a', text=True), ['First tag.', 'Nested tag.'])
+
+        self.assertSelects(
+            self.tree.find_all('a', text=re.compile("tag")),
+            ['First tag.', 'Nested tag.'])
+
+
+    def test_find_all_on_non_root_element(self):
+        # You can call find_all on any node, not just the root.
+        self.assertSelects(self.tree.c.find_all('a'), ['Nested tag.'])
+
+    def test_calling_element_invokes_find_all(self):
+        self.assertSelects(self.tree('a'), ['First tag.', 'Nested tag.'])
+
+    def test_find_all_by_tag_strainer(self):
+        self.assertSelects(
+            self.tree.find_all(SoupStrainer('a')),
+            ['First tag.', 'Nested tag.'])
+
+    def test_find_all_by_tag_names(self):
+        self.assertSelects(
+            self.tree.find_all(['a', 'b']),
+            ['First tag.', 'Second tag.', 'Nested tag.'])
+
+    def test_find_all_by_tag_dict(self):
+        self.assertSelects(
+            self.tree.find_all({'a' : True, 'b' : True}),
+            ['First tag.', 'Second tag.', 'Nested tag.'])
+
+    def test_find_all_by_tag_re(self):
+        self.assertSelects(
+            self.tree.find_all(re.compile('^[ab]$')),
+            ['First tag.', 'Second tag.', 'Nested tag.'])
+
+    def test_find_all_with_tags_matching_method(self):
+        # You can define an oracle method that determines whether
+        # a tag matches the search.
+        def id_matches_name(tag):
+            return tag.name == tag.get('id')
+
+        tree = self.soup("""Match 1.
+                            Does not match.
+                            Match 2.""")
+
+        self.assertSelects(
+            tree.find_all(id_matches_name), ["Match 1.", "Match 2."])
+
+
+class TestFindAllByAttribute(TreeTest):
+
+    def test_find_all_by_attribute_name(self):
+        # You can pass in keyword arguments to find_all to search by
+        # attribute.
+        tree = self.soup("""
+                         Matching a.
+                         
+                          Non-matching Matching b.a.
+                         """)
+        self.assertSelects(tree.find_all(id='first'),
+                           ["Matching a.", "Matching b."])
+
+    def test_find_all_by_utf8_attribute_value(self):
+        peace = u"×ולש".encode("utf8")
+        data = u''.encode("utf8")
+        soup = self.soup(data)
+        self.assertEqual([soup.a], soup.find_all(title=peace))
+        self.assertEqual([soup.a], soup.find_all(title=peace.decode("utf8")))
+        self.assertEqual([soup.a], soup.find_all(title=[peace, "something else"]))
+
+    def test_find_all_by_attribute_dict(self):
+        # You can pass in a dictionary as the argument 'attrs'. This
+        # lets you search for attributes like 'name' (a fixed argument
+        # to find_all) and 'class' (a reserved word in Python.)
+        tree = self.soup("""
+                         Name match.
+                         Class match.
+                         Non-match.
+                         A tag called 'name1'.
+                         """)
+
+        # This doesn't do what you want.
+        self.assertSelects(tree.find_all(name='name1'),
+                           ["A tag called 'name1'."])
+        # This does what you want.
+        self.assertSelects(tree.find_all(attrs={'name' : 'name1'}),
+                           ["Name match."])
+
+        self.assertSelects(tree.find_all(attrs={'class' : 'class2'}),
+                           ["Class match."])
+
+    def test_find_all_by_class(self):
+        tree = self.soup("""
+                         Class 1.
+                         Class 2.
+                         Class 1.
+                         Class 3 and 4.
+                         """)
+
+        # Passing in the class_ keyword argument will search against
+        # the 'class' attribute.
+        self.assertSelects(tree.find_all('a', class_='1'), ['Class 1.'])
+        self.assertSelects(tree.find_all('c', class_='3'), ['Class 3 and 4.'])
+        self.assertSelects(tree.find_all('c', class_='4'), ['Class 3 and 4.'])
+
+        # Passing in a string to 'attrs' will also search the CSS class.
+        self.assertSelects(tree.find_all('a', '1'), ['Class 1.'])
+        self.assertSelects(tree.find_all(attrs='1'), ['Class 1.', 'Class 1.'])
+        self.assertSelects(tree.find_all('c', '3'), ['Class 3 and 4.'])
+        self.assertSelects(tree.find_all('c', '4'), ['Class 3 and 4.'])
+
+    def test_find_by_class_when_multiple_classes_present(self):
+        tree = self.soup("Found it")
+
+        f = tree.find_all("gar", class_=re.compile("o"))
+        self.assertSelects(f, ["Found it"])
+
+        f = tree.find_all("gar", class_=re.compile("a"))
+        self.assertSelects(f, ["Found it"])
+
+        # Since the class is not the string "foo bar", but the two
+        # strings "foo" and "bar", this will not find anything.
+        f = tree.find_all("gar", class_=re.compile("o b"))
+        self.assertSelects(f, [])
+
+    def test_find_all_with_non_dictionary_for_attrs_finds_by_class(self):
+        soup = self.soup("Found it")
+
+        self.assertSelects(soup.find_all("a", re.compile("ba")), ["Found it"])
+
+        def big_attribute_value(value):
+            return len(value) > 3
+
+        self.assertSelects(soup.find_all("a", big_attribute_value), [])
+
+        def small_attribute_value(value):
+            return len(value) <= 3
+
+        self.assertSelects(
+            soup.find_all("a", small_attribute_value), ["Found it"])
+
+    def test_find_all_with_string_for_attrs_finds_multiple_classes(self):
+        soup = self.soup('')
+        a, a2 = soup.find_all("a")
+        self.assertEqual([a, a2], soup.find_all("a", "foo"))
+        self.assertEqual([a], soup.find_all("a", "bar"))
+
+        # If you specify the class as a string that contains a
+        # space, only that specific value will be found.
+        self.assertEqual([a], soup.find_all("a", class_="foo bar"))
+        self.assertEqual([a], soup.find_all("a", "foo bar"))
+        self.assertEqual([], soup.find_all("a", "bar foo"))
+
+    def test_find_all_by_attribute_soupstrainer(self):
+        tree = self.soup("""
+                         Match.
+                         Non-match.""")
+
+        strainer = SoupStrainer(attrs={'id' : 'first'})
+        self.assertSelects(tree.find_all(strainer), ['Match.'])
+
+    def test_find_all_with_missing_atribute(self):
+        # You can pass in None as the value of an attribute to find_all.
+        # This will match tags that do not have that attribute set.
+        tree = self.soup("""ID present.
+                            No ID present.
+                            ID is empty.""")
+        self.assertSelects(tree.find_all('a', id=None), ["No ID present."])
+
+    def test_find_all_with_defined_attribute(self):
+        # You can pass in None as the value of an attribute to find_all.
+        # This will match tags that have that attribute set to any value.
+        tree = self.soup("""ID present.
+                            No ID present.
+                            ID is empty.""")
+        self.assertSelects(
+            tree.find_all(id=True), ["ID present.", "ID is empty."])
+
+    def test_find_all_with_numeric_attribute(self):
+        # If you search for a number, it's treated as a string.
+        tree = self.soup("""Unquoted attribute.
+                            Quoted attribute.""")
+
+        expected = ["Unquoted attribute.", "Quoted attribute."]
+        self.assertSelects(tree.find_all(id=1), expected)
+        self.assertSelects(tree.find_all(id="1"), expected)
+
+    def test_find_all_with_list_attribute_values(self):
+        # You can pass a list of attribute values instead of just one,
+        # and you'll get tags that match any of the values.
+        tree = self.soup("""1
+                            2
+                            3
+                            No ID.""")
+        self.assertSelects(tree.find_all(id=["1", "3", "4"]),
+                           ["1", "3"])
+
+    def test_find_all_with_regular_expression_attribute_value(self):
+        # You can pass a regular expression as an attribute value, and
+        # you'll get tags whose values for that attribute match the
+        # regular expression.
+        tree = self.soup("""One a.
+                            Two as.
+                            Mixed as and bs.
+                            One b.
+                            No ID.""")
+
+        self.assertSelects(tree.find_all(id=re.compile("^a+$")),
+                           ["One a.", "Two as."])
+
+    def test_find_by_name_and_containing_string(self):
+        soup = self.soup("foobarfoo")
+        a = soup.a
+
+        self.assertEqual([a], soup.find_all("a", text="foo"))
+        self.assertEqual([], soup.find_all("a", text="bar"))
+        self.assertEqual([], soup.find_all("a", text="bar"))
+
+    def test_find_by_name_and_containing_string_when_string_is_buried(self):
+        soup = self.soup("foofoo")
+        self.assertEqual(soup.find_all("a"), soup.find_all("a", text="foo"))
+
+    def test_find_by_attribute_and_containing_string(self):
+        soup = self.soup('foofoo')
+        a = soup.a
+
+        self.assertEqual([a], soup.find_all(id=2, text="foo"))
+        self.assertEqual([], soup.find_all(id=1, text="bar"))
+
+
+
+
+class TestIndex(TreeTest):
+    """Test Tag.index"""
+    def test_index(self):
+        tree = self.soup("""

+                            Identical
+                            Not identical
+                            Identical
+
+                            Identical with child
+                            Also not identical
+                            Identical with child
+                            
""")
+        div = tree.div
+        for i, element in enumerate(div.contents):
+            self.assertEqual(i, div.index(element))
+        self.assertRaises(ValueError, tree.index, 1)
+
+
+class TestParentOperations(TreeTest):
+    """Test navigation and searching through an element's parents."""
+
+    def setUp(self):
+        super(TestParentOperations, self).setUp()
+        self.tree = self.soup('''
    
+                                 
      
+                                  
        
+                                   
          
+                                    Start here
+                                   
        
+                                  
      ''')
+        self.start = self.tree.b
+
+
+    def test_parent(self):
+        self.assertEqual(self.start.parent['id'], 'bottom')
+        self.assertEqual(self.start.parent.parent['id'], 'middle')
+        self.assertEqual(self.start.parent.parent.parent['id'], 'top')
+
+    def test_parent_of_top_tag_is_soup_object(self):
+        top_tag = self.tree.contents[0]
+        self.assertEqual(top_tag.parent, self.tree)
+
+    def test_soup_object_has_no_parent(self):
+        self.assertEqual(None, self.tree.parent)
+
+    def test_find_parents(self):
+        self.assertSelectsIDs(
+            self.start.find_parents('ul'), ['bottom', 'middle', 'top'])
+        self.assertSelectsIDs(
+            self.start.find_parents('ul', id="middle"), ['middle'])
+
+    def test_find_parent(self):
+        self.assertEqual(self.start.find_parent('ul')['id'], 'bottom')
+        self.assertEqual(self.start.find_parent('ul', id='top')['id'], 'top')
+
+    def test_parent_of_text_element(self):
+        text = self.tree.find(text="Start here")
+        self.assertEqual(text.parent.name, 'b')
+
+    def test_text_element_find_parent(self):
+        text = self.tree.find(text="Start here")
+        self.assertEqual(text.find_parent('ul')['id'], 'bottom')
+
+    def test_parent_generator(self):
+        parents = [parent['id'] for parent in self.start.parents
+                   if parent is not None and 'id' in parent.attrs]
+        self.assertEqual(parents, ['bottom', 'middle', 'top'])
+
+
+class ProximityTest(TreeTest):
+
+    def setUp(self):
+        super(TreeTest, self).setUp()
+        self.tree = self.soup(
+            'OneTwoThree')
+
+
+class TestNextOperations(ProximityTest):
+
+    def setUp(self):
+        super(TestNextOperations, self).setUp()
+        self.start = self.tree.b
+
+    def test_next(self):
+        self.assertEqual(self.start.next_element, "One")
+        self.assertEqual(self.start.next_element.next_element['id'], "2")
+
+    def test_next_of_last_item_is_none(self):
+        last = self.tree.find(text="Three")
+        self.assertEqual(last.next_element, None)
+
+    def test_next_of_root_is_none(self):
+        # The document root is outside the next/previous chain.
+        self.assertEqual(self.tree.next_element, None)
+
+    def test_find_all_next(self):
+        self.assertSelects(self.start.find_all_next('b'), ["Two", "Three"])
+        self.start.find_all_next(id=3)
+        self.assertSelects(self.start.find_all_next(id=3), ["Three"])
+
+    def test_find_next(self):
+        self.assertEqual(self.start.find_next('b')['id'], '2')
+        self.assertEqual(self.start.find_next(text="Three"), "Three")
+
+    def test_find_next_for_text_element(self):
+        text = self.tree.find(text="One")
+        self.assertEqual(text.find_next("b").string, "Two")
+        self.assertSelects(text.find_all_next("b"), ["Two", "Three"])
+
+    def test_next_generator(self):
+        start = self.tree.find(text="Two")
+        successors = [node for node in start.next_elements]
+        # There are two successors: the final  tag and its text contents.
+        tag, contents = successors
+        self.assertEqual(tag['id'], '3')
+        self.assertEqual(contents, "Three")
+
+class TestPreviousOperations(ProximityTest):
+
+    def setUp(self):
+        super(TestPreviousOperations, self).setUp()
+        self.end = self.tree.find(text="Three")
+
+    def test_previous(self):
+        self.assertEqual(self.end.previous_element['id'], "3")
+        self.assertEqual(self.end.previous_element.previous_element, "Two")
+
+    def test_previous_of_first_item_is_none(self):
+        first = self.tree.find('html')
+        self.assertEqual(first.previous_element, None)
+
+    def test_previous_of_root_is_none(self):
+        # The document root is outside the next/previous chain.
+        # XXX This is broken!
+        #self.assertEqual(self.tree.previous_element, None)
+        pass
+
+    def test_find_all_previous(self):
+        # The  tag containing the "Three" node is the predecessor
+        # of the "Three" node itself, which is why "Three" shows up
+        # here.
+        self.assertSelects(
+            self.end.find_all_previous('b'), ["Three", "Two", "One"])
+        self.assertSelects(self.end.find_all_previous(id=1), ["One"])
+
+    def test_find_previous(self):
+        self.assertEqual(self.end.find_previous('b')['id'], '3')
+        self.assertEqual(self.end.find_previous(text="One"), "One")
+
+    def test_find_previous_for_text_element(self):
+        text = self.tree.find(text="Three")
+        self.assertEqual(text.find_previous("b").string, "Three")
+        self.assertSelects(
+            text.find_all_previous("b"), ["Three", "Two", "One"])
+
+    def test_previous_generator(self):
+        start = self.tree.find(text="One")
+        predecessors = [node for node in start.previous_elements]
+
+        # There are four predecessors: the  tag containing "One"
+        # the  tag, the  tag, and the  tag.
+        b, body, head, html = predecessors
+        self.assertEqual(b['id'], '1')
+        self.assertEqual(body.name, "body")
+        self.assertEqual(head.name, "head")
+        self.assertEqual(html.name, "html")
+
+
+class SiblingTest(TreeTest):
+
+    def setUp(self):
+        super(SiblingTest, self).setUp()
+        markup = '''
+                    
+                     
+                    
+                    
+                     
+                    
+                    
+                     
+                    
+                    
+                    '''
+        # All that whitespace looks good but makes the tests more
+        # difficult. Get rid of it.
+        markup = re.compile("\n\s*").sub("", markup)
+        self.tree = self.soup(markup)
+
+
+class TestNextSibling(SiblingTest):
+
+    def setUp(self):
+        super(TestNextSibling, self).setUp()
+        self.start = self.tree.find(id="1")
+
+    def test_next_sibling_of_root_is_none(self):
+        self.assertEqual(self.tree.next_sibling, None)
+
+    def test_next_sibling(self):
+        self.assertEqual(self.start.next_sibling['id'], '2')
+        self.assertEqual(self.start.next_sibling.next_sibling['id'], '3')
+
+        # Note the difference between next_sibling and next_element.
+        self.assertEqual(self.start.next_element['id'], '1.1')
+
+    def test_next_sibling_may_not_exist(self):
+        self.assertEqual(self.tree.html.next_sibling, None)
+
+        nested_span = self.tree.find(id="1.1")
+        self.assertEqual(nested_span.next_sibling, None)
+
+        last_span = self.tree.find(id="4")
+        self.assertEqual(last_span.next_sibling, None)
+
+    def test_find_next_sibling(self):
+        self.assertEqual(self.start.find_next_sibling('span')['id'], '2')
+
+    def test_next_siblings(self):
+        self.assertSelectsIDs(self.start.find_next_siblings("span"),
+                              ['2', '3', '4'])
+
+        self.assertSelectsIDs(self.start.find_next_siblings(id='3'), ['3'])
+
+    def test_next_sibling_for_text_element(self):
+        soup = self.soup("Foobarbaz")
+        start = soup.find(text="Foo")
+        self.assertEqual(start.next_sibling.name, 'b')
+        self.assertEqual(start.next_sibling.next_sibling, 'baz')
+
+        self.assertSelects(start.find_next_siblings('b'), ['bar'])
+        self.assertEqual(start.find_next_sibling(text="baz"), "baz")
+        self.assertEqual(start.find_next_sibling(text="nonesuch"), None)
+
+
+class TestPreviousSibling(SiblingTest):
+
+    def setUp(self):
+        super(TestPreviousSibling, self).setUp()
+        self.end = self.tree.find(id="4")
+
+    def test_previous_sibling_of_root_is_none(self):
+        self.assertEqual(self.tree.previous_sibling, None)
+
+    def test_previous_sibling(self):
+        self.assertEqual(self.end.previous_sibling['id'], '3')
+        self.assertEqual(self.end.previous_sibling.previous_sibling['id'], '2')
+
+        # Note the difference between previous_sibling and previous_element.
+        self.assertEqual(self.end.previous_element['id'], '3.1')
+
+    def test_previous_sibling_may_not_exist(self):
+        self.assertEqual(self.tree.html.previous_sibling, None)
+
+        nested_span = self.tree.find(id="1.1")
+        self.assertEqual(nested_span.previous_sibling, None)
+
+        first_span = self.tree.find(id="1")
+        self.assertEqual(first_span.previous_sibling, None)
+
+    def test_find_previous_sibling(self):
+        self.assertEqual(self.end.find_previous_sibling('span')['id'], '3')
+
+    def test_previous_siblings(self):
+        self.assertSelectsIDs(self.end.find_previous_siblings("span"),
+                              ['3', '2', '1'])
+
+        self.assertSelectsIDs(self.end.find_previous_siblings(id='1'), ['1'])
+
+    def test_previous_sibling_for_text_element(self):
+        soup = self.soup("Foobarbaz")
+        start = soup.find(text="baz")
+        self.assertEqual(start.previous_sibling.name, 'b')
+        self.assertEqual(start.previous_sibling.previous_sibling, 'Foo')
+
+        self.assertSelects(start.find_previous_siblings('b'), ['bar'])
+        self.assertEqual(start.find_previous_sibling(text="Foo"), "Foo")
+        self.assertEqual(start.find_previous_sibling(text="nonesuch"), None)
+
+
+class TestTagCreation(SoupTest):
+    """Test the ability to create new tags."""
+    def test_new_tag(self):
+        soup = self.soup("")
+        new_tag = soup.new_tag("foo", bar="baz")
+        self.assertTrue(isinstance(new_tag, Tag))
+        self.assertEqual("foo", new_tag.name)
+        self.assertEqual(dict(bar="baz"), new_tag.attrs)
+        self.assertEqual(None, new_tag.parent)
+
+    def test_tag_inherits_self_closing_rules_from_builder(self):
+        if XML_BUILDER_PRESENT:
+            xml_soup = BeautifulSoup("", "xml")
+            xml_br = xml_soup.new_tag("br")
+            xml_p = xml_soup.new_tag("p")
+
+            # Both the 
       and 
       tag are empty-element, just because
+            # they have no contents.
+            self.assertEqual(b"
      ", xml_br.encode())
+            self.assertEqual(b"
      ", xml_p.encode())
+
+        html_soup = BeautifulSoup("", "html")
+        html_br = html_soup.new_tag("br")
+        html_p = html_soup.new_tag("p")
+
+        # The HTML builder users HTML's rules about which tags are
+        # empty-element tags, and the new tags reflect these rules.
+        self.assertEqual(b"
      ", html_br.encode())
+        self.assertEqual(b"
      ", html_p.encode())
+
+    def test_new_string_creates_navigablestring(self):
+        soup = self.soup("")
+        s = soup.new_string("foo")
+        self.assertEqual("foo", s)
+        self.assertTrue(isinstance(s, NavigableString))
+
+    def test_new_string_can_create_navigablestring_subclass(self):
+        soup = self.soup("")
+        s = soup.new_string("foo", Comment)
+        self.assertEqual("foo", s)
+        self.assertTrue(isinstance(s, Comment))
+
+class TestTreeModification(SoupTest):
+
+    def test_attribute_modification(self):
+        soup = self.soup('')
+        soup.a['id'] = 2
+        self.assertEqual(soup.decode(), self.document_for(''))
+        del(soup.a['id'])
+        self.assertEqual(soup.decode(), self.document_for(''))
+        soup.a['id2'] = 'foo'
+        self.assertEqual(soup.decode(), self.document_for(''))
+
+    def test_new_tag_creation(self):
+        builder = builder_registry.lookup('html')()
+        soup = self.soup("", builder=builder)
+        a = Tag(soup, builder, 'a')
+        ol = Tag(soup, builder, 'ol')
+        a['href'] = 'http://foo.com/'
+        soup.body.insert(0, a)
+        soup.body.insert(1, ol)
+        self.assertEqual(
+            soup.body.encode(),
+            b'
        ')
+
+    def test_append_to_contents_moves_tag(self):
+        doc = """
        Don't leave me here.
        
+                
        Don\'t leave!
        """
+        soup = self.soup(doc)
+        second_para = soup.find(id='2')
+        bold = soup.b
+
+        # Move the  tag to the end of the second paragraph.
+        soup.find(id='2').append(soup.b)
+
+        # The  tag is now a child of the second paragraph.
+        self.assertEqual(bold.parent, second_para)
+
+        self.assertEqual(
+            soup.decode(), self.document_for(
+                '
        Don\'t leave me .
        \n'
+                '
        Don\'t leave!here
        '))
+
+    def test_replace_with_returns_thing_that_was_replaced(self):
+        text = ""
+        soup = self.soup(text)
+        a = soup.a
+        new_a = a.replace_with(soup.c)
+        self.assertEqual(a, new_a)
+
+    def test_unwrap_returns_thing_that_was_replaced(self):
+        text = ""
+        soup = self.soup(text)
+        a = soup.a
+        new_a = a.unwrap()
+        self.assertEqual(a, new_a)
+
+    def test_replace_tag_with_itself(self):
+        text = "Foo"
+        soup = self.soup(text)
+        c = soup.c
+        soup.c.replace_with(c)
+        self.assertEqual(soup.decode(), self.document_for(text))
+
+    def test_replace_tag_with_its_parent_raises_exception(self):
+        text = ""
+        soup = self.soup(text)
+        self.assertRaises(ValueError, soup.b.replace_with, soup.a)
+
+    def test_insert_tag_into_itself_raises_exception(self):
+        text = ""
+        soup = self.soup(text)
+        self.assertRaises(ValueError, soup.a.insert, 0, soup.a)
+
+    def test_replace_with_maintains_next_element_throughout(self):
+        soup = self.soup('
        onethree
        ')
+        a = soup.a
+        b = a.contents[0]
+        # Make it so the  tag has two text children.
+        a.insert(1, "two")
+
+        # Now replace each one with the empty string.
+        left, right = a.contents
+        left.replaceWith('')
+        right.replaceWith('')
+
+        # The  tag is still connected to the tree.
+        self.assertEqual("three", soup.b.string)
+
+    def test_replace_final_node(self):
+        soup = self.soup("Argh!")
+        soup.find(text="Argh!").replace_with("Hooray!")
+        new_text = soup.find(text="Hooray!")
+        b = soup.b
+        self.assertEqual(new_text.previous_element, b)
+        self.assertEqual(new_text.parent, b)
+        self.assertEqual(new_text.previous_element.next_element, new_text)
+        self.assertEqual(new_text.next_element, None)
+
+    def test_consecutive_text_nodes(self):
+        # A builder should never create two consecutive text nodes,
+        # but if you insert one next to another, Beautiful Soup will
+        # handle it correctly.
+        soup = self.soup("Argh!")
+        soup.b.insert(1, "Hooray!")
+
+        self.assertEqual(
+            soup.decode(), self.document_for(
+                "Argh!Hooray!"))
+
+        new_text = soup.find(text="Hooray!")
+        self.assertEqual(new_text.previous_element, "Argh!")
+        self.assertEqual(new_text.previous_element.next_element, new_text)
+
+        self.assertEqual(new_text.previous_sibling, "Argh!")
+        self.assertEqual(new_text.previous_sibling.next_sibling, new_text)
+
+        self.assertEqual(new_text.next_sibling, None)
+        self.assertEqual(new_text.next_element, soup.c)
+
+    def test_insert_string(self):
+        soup = self.soup("")
+        soup.a.insert(0, "bar")
+        soup.a.insert(0, "foo")
+        # The string were added to the tag.
+        self.assertEqual(["foo", "bar"], soup.a.contents)
+        # And they were converted to NavigableStrings.
+        self.assertEqual(soup.a.contents[0].next_element, "bar")
+
+    def test_insert_tag(self):
+        builder = self.default_builder
+        soup = self.soup(
+            "Findlady!", builder=builder)
+        magic_tag = Tag(soup, builder, 'magictag')
+        magic_tag.insert(0, "the")
+        soup.a.insert(1, magic_tag)
+
+        self.assertEqual(
+            soup.decode(), self.document_for(
+                "Findthelady!"))
+
+        # Make sure all the relationships are hooked up correctly.
+        b_tag = soup.b
+        self.assertEqual(b_tag.next_sibling, magic_tag)
+        self.assertEqual(magic_tag.previous_sibling, b_tag)
+
+        find = b_tag.find(text="Find")
+        self.assertEqual(find.next_element, magic_tag)
+        self.assertEqual(magic_tag.previous_element, find)
+
+        c_tag = soup.c
+        self.assertEqual(magic_tag.next_sibling, c_tag)
+        self.assertEqual(c_tag.previous_sibling, magic_tag)
+
+        the = magic_tag.find(text="the")
+        self.assertEqual(the.parent, magic_tag)
+        self.assertEqual(the.next_element, c_tag)
+        self.assertEqual(c_tag.previous_element, the)
+
+    def test_append_child_thats_already_at_the_end(self):
+        data = ""
+        soup = self.soup(data)
+        soup.a.append(soup.b)
+        self.assertEqual(data, soup.decode())
+
+    def test_move_tag_to_beginning_of_parent(self):
+        data = ""
+        soup = self.soup(data)
+        soup.a.insert(0, soup.d)
+        self.assertEqual("", soup.decode())
+
+    def test_insert_works_on_empty_element_tag(self):
+        # This is a little strange, since most HTML parsers don't allow
+        # markup like this to come through. But in general, we don't
+        # know what the parser would or wouldn't have allowed, so
+        # I'm letting this succeed for now.
+        soup = self.soup("
        ")
+        soup.br.insert(1, "Contents")
+        self.assertEqual(str(soup.br), "
        Contents
        ")
+
+    def test_insert_before(self):
+        soup = self.soup("foobar")
+        soup.b.insert_before("BAZ")
+        soup.a.insert_before("QUUX")
+        self.assertEqual(
+            soup.decode(), self.document_for("QUUXfooBAZbar"))
+
+        soup.a.insert_before(soup.b)
+        self.assertEqual(
+            soup.decode(), self.document_for("QUUXbarfooBAZ"))
+
+    def test_insert_after(self):
+        soup = self.soup("foobar")
+        soup.b.insert_after("BAZ")
+        soup.a.insert_after("QUUX")
+        self.assertEqual(
+            soup.decode(), self.document_for("fooQUUXbarBAZ"))
+        soup.b.insert_after(soup.a)
+        self.assertEqual(
+            soup.decode(), self.document_for("QUUXbarfooBAZ"))
+
+    def test_insert_after_raises_exception_if_after_has_no_meaning(self):
+        soup = self.soup("")
+        tag = soup.new_tag("a")
+        string = soup.new_string("")
+        self.assertRaises(ValueError, string.insert_after, tag)
+        self.assertRaises(NotImplementedError, soup.insert_after, tag)
+        self.assertRaises(ValueError, tag.insert_after, tag)
+
+    def test_insert_before_raises_notimplementederror_if_before_has_no_meaning(self):
+        soup = self.soup("")
+        tag = soup.new_tag("a")
+        string = soup.new_string("")
+        self.assertRaises(ValueError, string.insert_before, tag)
+        self.assertRaises(NotImplementedError, soup.insert_before, tag)
+        self.assertRaises(ValueError, tag.insert_before, tag)
+
+    def test_replace_with(self):
+        soup = self.soup(
+                "
        There's no business like show business
        ")
+        no, show = soup.find_all('b')
+        show.replace_with(no)
+        self.assertEqual(
+            soup.decode(),
+            self.document_for(
+                "
        There's  business like no business
        "))
+
+        self.assertEqual(show.parent, None)
+        self.assertEqual(no.parent, soup.p)
+        self.assertEqual(no.next_element, "no")
+        self.assertEqual(no.next_sibling, " business")
+
+    def test_replace_first_child(self):
+        data = ""
+        soup = self.soup(data)
+        soup.b.replace_with(soup.c)
+        self.assertEqual("", soup.decode())
+
+    def test_replace_last_child(self):
+        data = ""
+        soup = self.soup(data)
+        soup.c.replace_with(soup.b)
+        self.assertEqual("", soup.decode())
+
+    def test_nested_tag_replace_with(self):
+        soup = self.soup(
+            """Wereservetherighttorefuseservice""")
+
+        # Replace the entire  tag and its contents ("reserve the
+        # right") with the  tag ("refuse").
+        remove_tag = soup.b
+        move_tag = soup.f
+        remove_tag.replace_with(move_tag)
+
+        self.assertEqual(
+            soup.decode(), self.document_for(
+                "Werefusetoservice"))
+
+        # The  tag is now an orphan.
+        self.assertEqual(remove_tag.parent, None)
+        self.assertEqual(remove_tag.find(text="right").next_element, None)
+        self.assertEqual(remove_tag.previous_element, None)
+        self.assertEqual(remove_tag.next_sibling, None)
+        self.assertEqual(remove_tag.previous_sibling, None)
+
+        # The  tag is now connected to the  tag.
+        self.assertEqual(move_tag.parent, soup.a)
+        self.assertEqual(move_tag.previous_element, "We")
+        self.assertEqual(move_tag.next_element.next_element, soup.e)
+        self.assertEqual(move_tag.next_sibling, None)
+
+        # The gap where the  tag used to be has been mended, and
+        # the word "to" is now connected to the  tag.
+        to_text = soup.find(text="to")
+        g_tag = soup.g
+        self.assertEqual(to_text.next_element, g_tag)
+        self.assertEqual(to_text.next_sibling, g_tag)
+        self.assertEqual(g_tag.previous_element, to_text)
+        self.assertEqual(g_tag.previous_sibling, to_text)
+
+    def test_unwrap(self):
+        tree = self.soup("""
+            
        Unneeded formatting is unneeded
        
+            """)
+        tree.em.unwrap()
+        self.assertEqual(tree.em, None)
+        self.assertEqual(tree.p.text, "Unneeded formatting is unneeded")
+
+    def test_wrap(self):
+        soup = self.soup("I wish I was bold.")
+        value = soup.string.wrap(soup.new_tag("b"))
+        self.assertEqual(value.decode(), "I wish I was bold.")
+        self.assertEqual(
+            soup.decode(), self.document_for("I wish I was bold."))
+
+    def test_wrap_extracts_tag_from_elsewhere(self):
+        soup = self.soup("I wish I was bold.")
+        soup.b.next_sibling.wrap(soup.b)
+        self.assertEqual(
+            soup.decode(), self.document_for("I wish I was bold."))
+
+    def test_wrap_puts_new_contents_at_the_end(self):
+        soup = self.soup("I like being bold.I wish I was bold.")
+        soup.b.next_sibling.wrap(soup.b)
+        self.assertEqual(2, len(soup.b.contents))
+        self.assertEqual(
+            soup.decode(), self.document_for(
+                "I like being bold.I wish I was bold."))
+
+    def test_extract(self):
+        soup = self.soup(
+            'Some content.  More content.')
+
+        self.assertEqual(len(soup.body.contents), 3)
+        extracted = soup.find(id="nav").extract()
+
+        self.assertEqual(
+            soup.decode(), "Some content.  More content.")
+        self.assertEqual(extracted.decode(), '')
+
+        # The extracted tag is now an orphan.
+        self.assertEqual(len(soup.body.contents), 2)
+        self.assertEqual(extracted.parent, None)
+        self.assertEqual(extracted.previous_element, None)
+        self.assertEqual(extracted.next_element.next_element, None)
+
+        # The gap where the extracted tag used to be has been mended.
+        content_1 = soup.find(text="Some content. ")
+        content_2 = soup.find(text=" More content.")
+        self.assertEqual(content_1.next_element, content_2)
+        self.assertEqual(content_1.next_sibling, content_2)
+        self.assertEqual(content_2.previous_element, content_1)
+        self.assertEqual(content_2.previous_sibling, content_1)
+
+    def test_extract_distinguishes_between_identical_strings(self):
+        soup = self.soup("        foobar")
+        foo_1 = soup.a.string
+        bar_1 = soup.b.string
+        foo_2 = soup.new_string("foo")
+        bar_2 = soup.new_string("bar")
+        soup.a.append(foo_2)
+        soup.b.append(bar_2)
+
+        # Now there are two identical strings in the  tag, and two
+        # in the  tag. Let's remove the first "foo" and the second
+        # "bar".
+        foo_1.extract()
+        bar_2.extract()
+        self.assertEqual(foo_2, soup.a.string)
+        self.assertEqual(bar_2, soup.b.string)
+
+    def test_clear(self):
+        """Tag.clear()"""
+        soup = self.soup("
        String Italicized and another
        ")
+        # clear using extract()
+        a = soup.a
+        soup.p.clear()
+        self.assertEqual(len(soup.p.contents), 0)
+        self.assertTrue(hasattr(a, "contents"))
+
+        # clear using decompose()
+        em = a.em
+        a.clear(decompose=True)
+        self.assertEqual(0, len(em.contents))
+
+    def test_string_set(self):
+        """Tag.string = 'string'"""
+        soup = self.soup(" ")
+        soup.a.string = "foo"
+        self.assertEqual(soup.a.contents, ["foo"])
+        soup.b.string = "bar"
+        self.assertEqual(soup.b.contents, ["bar"])
+
+    def test_string_set_does_not_affect_original_string(self):
+        soup = self.soup("foobar")
+        soup.b.string = soup.c.string
+        self.assertEqual(soup.a.encode(), b"barbar")
+
+    def test_set_string_preserves_class_of_string(self):
+        soup = self.soup("")
+        cdata = CData("foo")
+        soup.a.string = cdata
+        self.assertTrue(isinstance(soup.a.string, CData))
+
+class TestElementObjects(SoupTest):
+    """Test various features of element objects."""
+
+    def test_len(self):
+        """The length of an element is its number of children."""
+        soup = self.soup("123")
+
+        # The BeautifulSoup object itself contains one element: the
+        #  tag.
+        self.assertEqual(len(soup.contents), 1)
+        self.assertEqual(len(soup), 1)
+
+        # The  tag contains three elements: the text node "1", the
+        #  tag, and the text node "3".
+        self.assertEqual(len(soup.top), 3)
+        self.assertEqual(len(soup.top.contents), 3)
+
+    def test_member_access_invokes_find(self):
+        """Accessing a Python member .foo invokes find('foo')"""
+        soup = self.soup('')
+        self.assertEqual(soup.b, soup.find('b'))
+        self.assertEqual(soup.b.i, soup.find('b').find('i'))
+        self.assertEqual(soup.a, None)
+
+    def test_deprecated_member_access(self):
+        soup = self.soup('')
+        with warnings.catch_warnings(record=True) as w:
+            tag = soup.bTag
+        self.assertEqual(soup.b, tag)
+        self.assertEqual(
+            '.bTag is deprecated, use .find("b") instead.',
+            str(w[0].message))
+
+    def test_has_attr(self):
+        """has_attr() checks for the presence of an attribute.
+
+        Please note note: has_attr() is different from
+        __in__. has_attr() checks the tag's attributes and __in__
+        checks the tag's chidlren.
+        """
+        soup = self.soup("")
+        self.assertTrue(soup.foo.has_attr('attr'))
+        self.assertFalse(soup.foo.has_attr('attr2'))
+
+
+    def test_attributes_come_out_in_alphabetical_order(self):
+        markup = ''
+        self.assertSoupEquals(markup, '')
+
+    def test_string(self):
+        # A tag that contains only a text node makes that node
+        # available as .string.
+        soup = self.soup("foo")
+        self.assertEqual(soup.b.string, 'foo')
+
+    def test_empty_tag_has_no_string(self):
+        # A tag with no children has no .stirng.
+        soup = self.soup("")
+        self.assertEqual(soup.b.string, None)
+
+    def test_tag_with_multiple_children_has_no_string(self):
+        # A tag with no children has no .string.
+        soup = self.soup("foo")
+        self.assertEqual(soup.b.string, None)
+
+        soup = self.soup("foobar        ")
+        self.assertEqual(soup.b.string, None)
+
+        # Even if all the children are strings, due to trickery,
+        # it won't work--but this would be a good optimization.
+        soup = self.soup("foo        ")
+        soup.a.insert(1, "bar")
+        self.assertEqual(soup.a.string, None)
+
+    def test_tag_with_recursive_string_has_string(self):
+        # A tag with a single child which has a .string inherits that
+        # .string.
+        soup = self.soup("foo")
+        self.assertEqual(soup.a.string, "foo")
+        self.assertEqual(soup.string, "foo")
+
+    def test_lack_of_string(self):
+        """Only a tag containing a single text node has a .string."""
+        soup = self.soup("feo")
+        self.assertFalse(soup.b.string)
+
+        soup = self.soup("")
+        self.assertFalse(soup.b.string)
+
+    def test_all_text(self):
+        """Tag.text and Tag.get_text(sep=u"") -> all child text, concatenated"""
+        soup = self.soup("ar    t ")
+        self.assertEqual(soup.a.text, "ar  t ")
+        self.assertEqual(soup.a.get_text(strip=True), "art")
+        self.assertEqual(soup.a.get_text(","), "a,r, , t ")
+        self.assertEqual(soup.a.get_text(",", strip=True), "a,r,t")
+
+    def test_get_text_ignores_comments(self):
+        soup = self.soup("foobar")
+        self.assertEqual(soup.get_text(), "foobar")
+
+        self.assertEqual(
+            soup.get_text(types=(NavigableString, Comment)), "fooIGNOREbar")
+        self.assertEqual(
+            soup.get_text(types=None), "fooIGNOREbar")
+
+    def test_all_strings_ignores_comments(self):
+        soup = self.soup("foobar")
+        self.assertEqual(['foo', 'bar'], list(soup.strings))
+
+class TestCDAtaListAttributes(SoupTest):
+
+    """Testing cdata-list attributes like 'class'.
+    """
+    def test_single_value_becomes_list(self):
+        soup = self.soup("")
+        self.assertEqual(["foo"],soup.a['class'])
+
+    def test_multiple_values_becomes_list(self):
+        soup = self.soup("")
+        self.assertEqual(["foo", "bar"], soup.a['class'])
+
+    def test_multiple_values_separated_by_weird_whitespace(self):
+        soup = self.soup("")
+        self.assertEqual(["foo", "bar", "baz"],soup.a['class'])
+
+    def test_attributes_joined_into_string_on_output(self):
+        soup = self.soup("")
+        self.assertEqual(b'', soup.a.encode())
+
+    def test_accept_charset(self):
+        soup = self.soup('
        ')
+        self.assertEqual(['ISO-8859-1', 'UTF-8'], soup.form['accept-charset'])
+
+    def test_cdata_attribute_applying_only_to_one_tag(self):
+        data = ''
+        soup = self.soup(data)
+        # We saw in another test that accept-charset is a cdata-list
+        # attribute for the  tag. But it's not a cdata-list
+        # attribute for any other tag.
+        self.assertEqual('ISO-8859-1 UTF-8', soup.a['accept-charset'])
+
+    def test_string_has_immutable_name_property(self):
+        string = self.soup("s").string
+        self.assertEqual(None, string.name)
+        def t():
+            string.name = 'foo'
+        self.assertRaises(AttributeError, t)
+
+class TestPersistence(SoupTest):
+    "Testing features like pickle and deepcopy."
+
+    def setUp(self):
+        super(TestPersistence, self).setUp()
+        self.page = """
+
+
+
+Beautiful Soup: We called him Tortoise because he taught us.
+
+
+
+
+
+
+foo
+bar
+
+"""
+        self.tree = self.soup(self.page)
+
+    def test_pickle_and_unpickle_identity(self):
+        # Pickling a tree, then unpickling it, yields a tree identical
+        # to the original.
+        dumped = pickle.dumps(self.tree, 2)
+        loaded = pickle.loads(dumped)
+        self.assertEqual(loaded.__class__, BeautifulSoup)
+        self.assertEqual(loaded.decode(), self.tree.decode())
+
+    def test_deepcopy_identity(self):
+        # Making a deepcopy of a tree yields an identical tree.
+        copied = copy.deepcopy(self.tree)
+        self.assertEqual(copied.decode(), self.tree.decode())
+
+    def test_unicode_pickle(self):
+        # A tree containing Unicode characters can be pickled.
+        html = u"\N{SNOWMAN}"
+        soup = self.soup(html)
+        dumped = pickle.dumps(soup, pickle.HIGHEST_PROTOCOL)
+        loaded = pickle.loads(dumped)
+        self.assertEqual(loaded.decode(), soup.decode())
+
+
+class TestSubstitutions(SoupTest):
+
+    def test_default_formatter_is_minimal(self):
+        markup = u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
+        soup = self.soup(markup)
+        decoded = soup.decode(formatter="minimal")
+        # The < is converted back into < but the e-with-acute is left alone.
+        self.assertEqual(
+            decoded,
+            self.document_for(
+                u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"))
+
+    def test_formatter_html(self):
+        markup = u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
+        soup = self.soup(markup)
+        decoded = soup.decode(formatter="html")
+        self.assertEqual(
+            decoded,
+            self.document_for("<<Sacré bleu!>>"))
+
+    def test_formatter_minimal(self):
+        markup = u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
+        soup = self.soup(markup)
+        decoded = soup.decode(formatter="minimal")
+        # The < is converted back into < but the e-with-acute is left alone.
+        self.assertEqual(
+            decoded,
+            self.document_for(
+                u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"))
+
+    def test_formatter_null(self):
+        markup = u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
+        soup = self.soup(markup)
+        decoded = soup.decode(formatter=None)
+        # Neither the angle brackets nor the e-with-acute are converted.
+        # This is not valid HTML, but it's what the user wanted.
+        self.assertEqual(decoded,
+                          self.document_for(u"<>"))
+
+    def test_formatter_custom(self):
+        markup = u"<foo>bar"
+        soup = self.soup(markup)
+        decoded = soup.decode(formatter = lambda x: x.upper())
+        # Instead of normal entity conversion code, the custom
+        # callable is called on every string.
+        self.assertEqual(
+            decoded,
+            self.document_for(u"BAR"))
+
+    def test_formatter_is_run_on_attribute_values(self):
+        markup = u'e'
+        soup = self.soup(markup)
+        a = soup.a
+
+        expect_minimal = u'e'
+
+        self.assertEqual(expect_minimal, a.decode())
+        self.assertEqual(expect_minimal, a.decode(formatter="minimal"))
+
+        expect_html = u'e'
+        self.assertEqual(expect_html, a.decode(formatter="html"))
+
+        self.assertEqual(markup, a.decode(formatter=None))
+        expect_upper = u'E'
+        self.assertEqual(expect_upper, a.decode(formatter=lambda x: x.upper()))
+
+    def test_formatter_skips_script_tag_for_html_documents(self):
+        doc = """
+  
+"""
+        encoded = BeautifulSoup(doc).encode()
+        self.assertTrue(b"< < hey > >" in encoded)
+
+    def test_formatter_skips_style_tag_for_html_documents(self):
+        doc = """
+  
+"""
+        encoded = BeautifulSoup(doc).encode()
+        self.assertTrue(b"< < hey > >" in encoded)
+
+    def test_prettify_leaves_preformatted_text_alone(self):
+        soup = self.soup("
          foo  
          \tbar\n  \n  
          baz  ")
+        # Everything outside the 
         tag is reformatted, but everything
+        # inside is left alone.
+        self.assertEqual(
+            u'
        \n foo\n 
          \tbar\n  \n  
        \n baz\n
        ',
+            soup.div.prettify())
+
+    def test_prettify_accepts_formatter(self):
+        soup = BeautifulSoup("foo")
+        pretty = soup.prettify(formatter = lambda x: x.upper())
+        self.assertTrue("FOO" in pretty)
+
+    def test_prettify_outputs_unicode_by_default(self):
+        soup = self.soup("")
+        self.assertEqual(unicode, type(soup.prettify()))
+
+    def test_prettify_can_encode_data(self):
+        soup = self.soup("")
+        self.assertEqual(bytes, type(soup.prettify("utf-8")))
+
+    def test_html_entity_substitution_off_by_default(self):
+        markup = u"Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!"
+        soup = self.soup(markup)
+        encoded = soup.b.encode("utf-8")
+        self.assertEqual(encoded, markup.encode('utf-8'))
+
+    def test_encoding_substitution(self):
+        # Here's the  tag saying that a document is
+        # encoded in Shift-JIS.
+        meta_tag = ('')
+        soup = self.soup(meta_tag)
+
+        # Parse the document, and the charset apprears unchanged.
+        self.assertEqual(soup.meta['content'], 'text/html; charset=x-sjis')
+
+        # Encode the document into some encoding, and the encoding is
+        # substituted into the meta tag.
+        utf_8 = soup.encode("utf-8")
+        self.assertTrue(b"charset=utf-8" in utf_8)
+
+        euc_jp = soup.encode("euc_jp")
+        self.assertTrue(b"charset=euc_jp" in euc_jp)
+
+        shift_jis = soup.encode("shift-jis")
+        self.assertTrue(b"charset=shift-jis" in shift_jis)
+
+        utf_16_u = soup.encode("utf-16").decode("utf-16")
+        self.assertTrue("charset=utf-16" in utf_16_u)
+
+    def test_encoding_substitution_doesnt_happen_if_tag_is_strained(self):
+        markup = ('
        foo
        ')
+
+        # Beautiful Soup used to try to rewrite the meta tag even if the
+        # meta tag got filtered out by the strainer. This test makes
+        # sure that doesn't happen.
+        strainer = SoupStrainer('pre')
+        soup = self.soup(markup, parse_only=strainer)
+        self.assertEqual(soup.contents[0].name, 'pre')
+
+class TestEncoding(SoupTest):
+    """Test the ability to encode objects into strings."""
+
+    def test_unicode_string_can_be_encoded(self):
+        html = u"\N{SNOWMAN}"
+        soup = self.soup(html)
+        self.assertEqual(soup.b.string.encode("utf-8"),
+                          u"\N{SNOWMAN}".encode("utf-8"))
+
+    def test_tag_containing_unicode_string_can_be_encoded(self):
+        html = u"\N{SNOWMAN}"
+        soup = self.soup(html)
+        self.assertEqual(
+            soup.b.encode("utf-8"), html.encode("utf-8"))
+
+    def test_encoding_substitutes_unrecognized_characters_by_default(self):
+        html = u"\N{SNOWMAN}"
+        soup = self.soup(html)
+        self.assertEqual(soup.b.encode("ascii"), b"☃")
+
+    def test_encoding_can_be_made_strict(self):
+        html = u"\N{SNOWMAN}"
+        soup = self.soup(html)
+        self.assertRaises(
+            UnicodeEncodeError, soup.encode, "ascii", errors="strict")
+
+    def test_decode_contents(self):
+        html = u"\N{SNOWMAN}"
+        soup = self.soup(html)
+        self.assertEqual(u"\N{SNOWMAN}", soup.b.decode_contents())
+
+    def test_encode_contents(self):
+        html = u"\N{SNOWMAN}"
+        soup = self.soup(html)
+        self.assertEqual(
+            u"\N{SNOWMAN}".encode("utf8"), soup.b.encode_contents(
+                encoding="utf8"))
+
+    def test_deprecated_renderContents(self):
+        html = u"\N{SNOWMAN}"
+        soup = self.soup(html)
+        self.assertEqual(
+            u"\N{SNOWMAN}".encode("utf8"), soup.b.renderContents())
+
+class TestNavigableStringSubclasses(SoupTest):
+
+    def test_cdata(self):
+        # None of the current builders turn CDATA sections into CData
+        # objects, but you can create them manually.
+        soup = self.soup("")
+        cdata = CData("foo")
+        soup.insert(1, cdata)
+        self.assertEqual(str(soup), "")
+        self.assertEqual(soup.find(text="foo"), "foo")
+        self.assertEqual(soup.contents[0], "foo")
+
+    def test_cdata_is_never_formatted(self):
+        """Text inside a CData object is passed into the formatter.
+
+        But the return value is ignored.
+        """
+
+        self.count = 0
+        def increment(*args):
+            self.count += 1
+            return "BITTER FAILURE"
+
+        soup = self.soup("")
+        cdata = CData("<><><>")
+        soup.insert(1, cdata)
+        self.assertEqual(
+            b"<><>]]>", soup.encode(formatter=increment))
+        self.assertEqual(1, self.count)
+
+    def test_doctype_ends_in_newline(self):
+        # Unlike other NavigableString subclasses, a DOCTYPE always ends
+        # in a newline.
+        doctype = Doctype("foo")
+        soup = self.soup("")
+        soup.insert(1, doctype)
+        self.assertEqual(soup.encode(), b"\n")
+
+
+class TestSoupSelector(TreeTest):
+
+    HTML = """
+
+
+
+The title
+
+
+
+
+
        
+
        
+
        An H1
        
+
        Some text
        
+
        Some more text
        
+
        An H2
        
+
        Another
        
+Bob
+
        Another H2
        
+me
+
+span1a1
+span1a2 test
+
+span2a1
+
+
+
+
        
+
        English
        
+
        English UK
        
+
        English US
        
+
        French
        
+
        
+
+
+"""
+
+    def setUp(self):
+        self.soup = BeautifulSoup(self.HTML)
+
+    def assertSelects(self, selector, expected_ids):
+        el_ids = [el['id'] for el in self.soup.select(selector)]
+        el_ids.sort()
+        expected_ids.sort()
+        self.assertEqual(expected_ids, el_ids,
+            "Selector %s, expected [%s], got [%s]" % (
+                selector, ', '.join(expected_ids), ', '.join(el_ids)
+            )
+        )
+
+    assertSelect = assertSelects
+
+    def assertSelectMultiple(self, *tests):
+        for selector, expected_ids in tests:
+            self.assertSelect(selector, expected_ids)
+
+    def test_one_tag_one(self):
+        els = self.soup.select('title')
+        self.assertEqual(len(els), 1)
+        self.assertEqual(els[0].name, 'title')
+        self.assertEqual(els[0].contents, [u'The title'])
+
+    def test_one_tag_many(self):
+        els = self.soup.select('div')
+        self.assertEqual(len(els), 3)
+        for div in els:
+            self.assertEqual(div.name, 'div')
+
+    def test_tag_in_tag_one(self):
+        els = self.soup.select('div div')
+        self.assertSelects('div div', ['inner'])
+
+    def test_tag_in_tag_many(self):
+        for selector in ('html div', 'html body div', 'body div'):
+            self.assertSelects(selector, ['main', 'inner', 'footer'])
+
+    def test_tag_no_match(self):
+        self.assertEqual(len(self.soup.select('del')), 0)
+
+    def test_invalid_tag(self):
+        self.assertRaises(ValueError, self.soup.select, 'tag%t')
+
+    def test_header_tags(self):
+        self.assertSelectMultiple(
+            ('h1', ['header1']),
+            ('h2', ['header2', 'header3']),
+        )
+
+    def test_class_one(self):
+        for selector in ('.onep', 'p.onep', 'html p.onep'):
+            els = self.soup.select(selector)
+            self.assertEqual(len(els), 1)
+            self.assertEqual(els[0].name, 'p')
+            self.assertEqual(els[0]['class'], ['onep'])
+
+    def test_class_mismatched_tag(self):
+        els = self.soup.select('div.onep')
+        self.assertEqual(len(els), 0)
+
+    def test_one_id(self):
+        for selector in ('div#inner', '#inner', 'div div#inner'):
+            self.assertSelects(selector, ['inner'])
+
+    def test_bad_id(self):
+        els = self.soup.select('#doesnotexist')
+        self.assertEqual(len(els), 0)
+
+    def test_items_in_id(self):
+        els = self.soup.select('div#inner p')
+        self.assertEqual(len(els), 3)
+        for el in els:
+            self.assertEqual(el.name, 'p')
+        self.assertEqual(els[1]['class'], ['onep'])
+        self.assertFalse(els[0].has_attr('class'))
+
+    def test_a_bunch_of_emptys(self):
+        for selector in ('div#main del', 'div#main div.oops', 'div div#main'):
+            self.assertEqual(len(self.soup.select(selector)), 0)
+
+    def test_multi_class_support(self):
+        for selector in ('.class1', 'p.class1', '.class2', 'p.class2',
+            '.class3', 'p.class3', 'html p.class2', 'div#inner .class2'):
+            self.assertSelects(selector, ['pmulti'])
+
+    def test_multi_class_selection(self):
+        for selector in ('.class1.class3', '.class3.class2',
+                         '.class1.class2.class3'):
+            self.assertSelects(selector, ['pmulti'])
+
+    def test_child_selector(self):
+        self.assertSelects('.s1 > a', ['s1a1', 's1a2'])
+        self.assertSelects('.s1 > a span', ['s1a2s1'])
+
+    def test_child_selector_id(self):
+        self.assertSelects('.s1 > a#s1a2 span', ['s1a2s1'])
+
+    def test_attribute_equals(self):
+        self.assertSelectMultiple(
+            ('p[class="onep"]', ['p1']),
+            ('p[id="p1"]', ['p1']),
+            ('[class="onep"]', ['p1']),
+            ('[id="p1"]', ['p1']),
+            ('link[rel="stylesheet"]', ['l1']),
+            ('link[type="text/css"]', ['l1']),
+            ('link[href="blah.css"]', ['l1']),
+            ('link[href="no-blah.css"]', []),
+            ('[rel="stylesheet"]', ['l1']),
+            ('[type="text/css"]', ['l1']),
+            ('[href="blah.css"]', ['l1']),
+            ('[href="no-blah.css"]', []),
+            ('p[href="no-blah.css"]', []),
+            ('[href="no-blah.css"]', []),
+        )
+
+    def test_attribute_tilde(self):
+        self.assertSelectMultiple(
+            ('p[class~="class1"]', ['pmulti']),
+            ('p[class~="class2"]', ['pmulti']),
+            ('p[class~="class3"]', ['pmulti']),
+            ('[class~="class1"]', ['pmulti']),
+            ('[class~="class2"]', ['pmulti']),
+            ('[class~="class3"]', ['pmulti']),
+            ('a[rel~="friend"]', ['bob']),
+            ('a[rel~="met"]', ['bob']),
+            ('[rel~="friend"]', ['bob']),
+            ('[rel~="met"]', ['bob']),
+        )
+
+    def test_attribute_startswith(self):
+        self.assertSelectMultiple(
+            ('[rel^="style"]', ['l1']),
+            ('link[rel^="style"]', ['l1']),
+            ('notlink[rel^="notstyle"]', []),
+            ('[rel^="notstyle"]', []),
+            ('link[rel^="notstyle"]', []),
+            ('link[href^="bla"]', ['l1']),
+            ('a[href^="http://"]', ['bob', 'me']),
+            ('[href^="http://"]', ['bob', 'me']),
+            ('[id^="p"]', ['pmulti', 'p1']),
+            ('[id^="m"]', ['me', 'main']),
+            ('div[id^="m"]', ['main']),
+            ('a[id^="m"]', ['me']),
+        )
+
+    def test_attribute_endswith(self):
+        self.assertSelectMultiple(
+            ('[href$=".css"]', ['l1']),
+            ('link[href$=".css"]', ['l1']),
+            ('link[id$="1"]', ['l1']),
+            ('[id$="1"]', ['l1', 'p1', 'header1', 's1a1', 's2a1', 's1a2s1']),
+            ('div[id$="1"]', []),
+            ('[id$="noending"]', []),
+        )
+
+    def test_attribute_contains(self):
+        self.assertSelectMultiple(
+            # From test_attribute_startswith
+            ('[rel*="style"]', ['l1']),
+            ('link[rel*="style"]', ['l1']),
+            ('notlink[rel*="notstyle"]', []),
+            ('[rel*="notstyle"]', []),
+            ('link[rel*="notstyle"]', []),
+            ('link[href*="bla"]', ['l1']),
+            ('a[href*="http://"]', ['bob', 'me']),
+            ('[href*="http://"]', ['bob', 'me']),
+            ('[id*="p"]', ['pmulti', 'p1']),
+            ('div[id*="m"]', ['main']),
+            ('a[id*="m"]', ['me']),
+            # From test_attribute_endswith
+            ('[href*=".css"]', ['l1']),
+            ('link[href*=".css"]', ['l1']),
+            ('link[id*="1"]', ['l1']),
+            ('[id*="1"]', ['l1', 'p1', 'header1', 's1a1', 's1a2', 's2a1', 's1a2s1']),
+            ('div[id*="1"]', []),
+            ('[id*="noending"]', []),
+            # New for this test
+            ('[href*="."]', ['bob', 'me', 'l1']),
+            ('a[href*="."]', ['bob', 'me']),
+            ('link[href*="."]', ['l1']),
+            ('div[id*="n"]', ['main', 'inner']),
+            ('div[id*="nn"]', ['inner']),
+        )
+
+    def test_attribute_exact_or_hypen(self):
+        self.assertSelectMultiple(
+            ('p[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']),
+            ('[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']),
+            ('p[lang|="fr"]', ['lang-fr']),
+            ('p[lang|="gb"]', []),
+        )
+
+    def test_attribute_exists(self):
+        self.assertSelectMultiple(
+            ('[rel]', ['l1', 'bob', 'me']),
+            ('link[rel]', ['l1']),
+            ('a[rel]', ['bob', 'me']),
+            ('[lang]', ['lang-en', 'lang-en-gb', 'lang-en-us', 'lang-fr']),
+            ('p[class]', ['p1', 'pmulti']),
+            ('[blah]', []),
+            ('p[blah]', []),
+        )
+
+    def test_nth_of_type(self):
+        # Try to select first paragraph
+        els = self.soup.select('div#inner p:nth-of-type(1)')
+        self.assertEqual(len(els), 1)
+        self.assertEqual(els[0].string, u'Some text')
+
+        # Try to select third paragraph
+        els = self.soup.select('div#inner p:nth-of-type(3)')
+        self.assertEqual(len(els), 1)
+        self.assertEqual(els[0].string, u'Another')
+
+        # Try to select (non-existent!) fourth paragraph
+        els = self.soup.select('div#inner p:nth-of-type(4)')
+        self.assertEqual(len(els), 0)
+
+        # Pass in an invalid value.
+        self.assertRaises(
+            ValueError, self.soup.select, 'div p:nth-of-type(0)')
+
+    def test_nth_of_type_direct_descendant(self):
+        els = self.soup.select('div#inner > p:nth-of-type(1)')
+        self.assertEqual(len(els), 1)
+        self.assertEqual(els[0].string, u'Some text')
+
+    def test_id_child_selector_nth_of_type(self):
+        self.assertSelects('#inner > p:nth-of-type(2)', ['p1'])
+
+    def test_select_on_element(self):
+        # Other tests operate on the tree; this operates on an element
+        # within the tree.
+        inner = self.soup.find("div", id="main")
+        selected = inner.select("div")
+        # The 
         tag was selected. The