From eb8dc40360f0cfef56fb6947cc817a547d6d9bc6 Mon Sep 17 00:00:00 2001 From: Dave Cobbley Date: Tue, 14 Aug 2018 10:05:37 -0700 Subject: [Subtree] Removing import-layers directory As part of the move to subtrees, need to bring all the import layers content to the top level. Change-Id: I4a163d10898cbc6e11c27f776f60e1a470049d8f Signed-off-by: Dave Cobbley Signed-off-by: Brad Bishop --- poky/bitbake/lib/toaster/__init__.py | 0 poky/bitbake/lib/toaster/bldcollector/__init__.py | 0 poky/bitbake/lib/toaster/bldcollector/admin.py | 33 + poky/bitbake/lib/toaster/bldcollector/urls.py | 27 + poky/bitbake/lib/toaster/bldcollector/views.py | 62 + poky/bitbake/lib/toaster/bldcontrol/__init__.py | 0 poky/bitbake/lib/toaster/bldcontrol/admin.py | 8 + .../bitbake/lib/toaster/bldcontrol/bbcontroller.py | 142 + .../toaster/bldcontrol/localhostbecontroller.py | 401 ++ .../lib/toaster/bldcontrol/management/__init__.py | 0 .../bldcontrol/management/commands/__init__.py | 0 .../management/commands/checksettings.py | 167 + .../bldcontrol/management/commands/runbuilds.py | 192 + .../toaster/bldcontrol/migrations/0001_initial.py | 113 + .../migrations/0002_auto_20160120_1250.py | 19 + .../migrations/0003_add_cancelling_state.py | 19 + .../migrations/0004_auto_20160523_1446.py | 34 + .../migrations/0005_reorder_buildrequest_states.py | 19 + .../migrations/0006_brlayer_local_source_dir.py | 19 + .../migrations/0007_brlayers_optional_gitinfo.py | 29 + .../lib/toaster/bldcontrol/migrations/__init__.py | 0 poky/bitbake/lib/toaster/bldcontrol/models.py | 162 + poky/bitbake/lib/toaster/bldcontrol/views.py | 1 + poky/bitbake/lib/toaster/manage.py | 10 + poky/bitbake/lib/toaster/orm/__init__.py | 0 poky/bitbake/lib/toaster/orm/fixtures/README | 30 + .../orm/fixtures/custom_toaster_append.sh_sample | 49 + poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml | 97 + poky/bitbake/lib/toaster/orm/fixtures/poky.xml | 234 + poky/bitbake/lib/toaster/orm/fixtures/settings.xml | 33 + .../bitbake/lib/toaster/orm/management/__init__.py | 0 .../toaster/orm/management/commands/__init__.py | 0 .../toaster/orm/management/commands/lsupdates.py | 337 + .../lib/toaster/orm/migrations/0001_initial.py | 504 ++ .../orm/migrations/0002_customimagerecipe.py | 24 + .../orm/migrations/0003_customimagepackage.py | 24 + .../lib/toaster/orm/migrations/0004_provides.py | 27 + .../orm/migrations/0005_task_field_separation.py | 48 + .../orm/migrations/0006_add_cancelled_state.py | 19 + .../orm/migrations/0007_auto_20160523_1446.py | 89 + .../migrations/0008_refactor_artifact_models.py | 39 + .../0009_target_package_manifest_path.py | 19 + .../0010_delete_layer_source_references.py | 118 + .../orm/migrations/0011_delete_layersource.py | 17 + .../0012_use_release_instead_of_up_branch.py | 62 + .../0013_recipe_parse_progress_fields.py | 24 + .../orm/migrations/0014_allow_empty_buildname.py | 19 + .../orm/migrations/0015_layer_local_source_dir.py | 19 + .../toaster/orm/migrations/0016_clone_progress.py | 24 + .../toaster/orm/migrations/0017_distro_clone.py | 25 + .../bitbake/lib/toaster/orm/migrations/__init__.py | 0 poky/bitbake/lib/toaster/orm/models.py | 1832 ++++++ poky/bitbake/lib/toaster/tests/__init__.py | 0 poky/bitbake/lib/toaster/tests/browser/README | 74 + poky/bitbake/lib/toaster/tests/browser/__init__.py | 0 .../lib/toaster/tests/browser/selenium_helpers.py | 34 + .../toaster/tests/browser/selenium_helpers_base.py | 227 + .../toaster/tests/browser/test_all_builds_page.py | 233 + .../tests/browser/test_all_projects_page.py | 217 + .../tests/browser/test_builddashboard_page.py | 347 + .../browser/test_builddashboard_page_artifacts.py | 222 + .../browser/test_builddashboard_page_recipes.py | 66 + .../browser/test_builddashboard_page_tasks.py | 65 + .../toaster/tests/browser/test_js_unit_tests.py | 57 + .../lib/toaster/tests/browser/test_landing_page.py | 108 + .../tests/browser/test_layerdetails_page.py | 216 + .../browser/test_most_recent_builds_states.py | 211 + .../tests/browser/test_new_custom_image_page.py | 161 + .../toaster/tests/browser/test_new_project_page.py | 113 + .../tests/browser/test_project_builds_page.py | 168 + .../tests/browser/test_project_config_page.py | 231 + .../lib/toaster/tests/browser/test_project_page.py | 59 + .../lib/toaster/tests/browser/test_sample.py | 41 + .../lib/toaster/tests/browser/test_task_page.py | 76 + .../toaster/tests/browser/test_toastertable_ui.py | 160 + poky/bitbake/lib/toaster/tests/builds/README | 14 + poky/bitbake/lib/toaster/tests/builds/__init__.py | 0 poky/bitbake/lib/toaster/tests/builds/buildtest.py | 169 + .../toaster/tests/builds/test_core_image_min.py | 386 ++ .../bitbake/lib/toaster/tests/commands/__init__.py | 0 .../lib/toaster/tests/commands/test_loaddata.py | 61 + .../lib/toaster/tests/commands/test_lsupdates.py | 45 + .../lib/toaster/tests/commands/test_runbuilds.py | 88 + poky/bitbake/lib/toaster/tests/db/__init__.py | 0 poky/bitbake/lib/toaster/tests/db/test_db.py | 55 + poky/bitbake/lib/toaster/tests/eventreplay/README | 22 + .../lib/toaster/tests/eventreplay/__init__.py | 97 + poky/bitbake/lib/toaster/tests/functional/README | 0 .../lib/toaster/tests/functional/__init__.py | 0 .../toaster/tests/functional/functional_helpers.py | 122 + .../tests/functional/test_functional_basic.py | 243 + .../toaster/tests/toaster-tests-requirements.txt | 1 + poky/bitbake/lib/toaster/tests/views/README | 4 + poky/bitbake/lib/toaster/tests/views/__init__.py | 0 poky/bitbake/lib/toaster/tests/views/test_views.py | 540 ++ poky/bitbake/lib/toaster/toastergui/__init__.py | 0 poky/bitbake/lib/toaster/toastergui/api.py | 1056 +++ poky/bitbake/lib/toaster/toastergui/buildtables.py | 609 ++ .../fixtures/toastergui-unittest-data.xml | 459 ++ .../toaster/toastergui/static/css/bootstrap.css | 6760 ++++++++++++++++++++ .../toastergui/static/css/bootstrap.css.map | 1 + .../toastergui/static/css/bootstrap.min.css | 6 + .../toastergui/static/css/bootstrap.min.css.map | 1 + .../static/css/bootstrap3-transition.css | 238 + .../lib/toaster/toastergui/static/css/default.css | 369 ++ .../toastergui/static/css/font-awesome.min.css | 33 + .../ui-bg_diagonals-thick_18_b81900_40x40.png | Bin 0 -> 418 bytes .../ui-bg_diagonals-thick_20_666666_40x40.png | Bin 0 -> 312 bytes .../css/images/ui-bg_flat_10_000000_40x100.png | Bin 0 -> 205 bytes .../css/images/ui-bg_glass_100_f6f6f6_1x400.png | Bin 0 -> 262 bytes .../css/images/ui-bg_glass_100_fdf5ce_1x400.png | Bin 0 -> 348 bytes .../css/images/ui-bg_glass_65_ffffff_1x400.png | Bin 0 -> 207 bytes .../images/ui-bg_gloss-wave_35_f6a828_500x100.png | Bin 0 -> 5815 bytes .../ui-bg_highlight-soft_100_eeeeee_1x100.png | Bin 0 -> 278 bytes .../ui-bg_highlight-soft_75_ffe45c_1x100.png | Bin 0 -> 328 bytes .../static/css/images/ui-icons_222222_256x240.png | Bin 0 -> 6922 bytes .../static/css/images/ui-icons_228ef1_256x240.png | Bin 0 -> 4549 bytes .../static/css/images/ui-icons_ef8c08_256x240.png | Bin 0 -> 4549 bytes .../static/css/images/ui-icons_ffd27a_256x240.png | Bin 0 -> 4549 bytes .../static/css/images/ui-icons_ffffff_256x240.png | Bin 0 -> 6299 bytes .../toastergui/static/css/jquery-ui.min.css | 7 + .../static/css/jquery-ui.structure.min.css | 5 + .../toastergui/static/css/jquery-ui.theme.min.css | 5 + .../toastergui/static/css/jquery.treetable.css | 28 + .../static/css/jquery.treetable.theme.default.css | 64 + .../static/css/jquery.treetable.theme.toaster.css | 38 + .../toaster/toastergui/static/css/qunit-1.18.0.css | 1 + .../lib/toaster/toastergui/static/css/screen.css | 28 + .../toastergui/static/fonts/FontAwesome.otf | Bin 0 -> 48748 bytes .../static/fonts/fontawesome-webfont.eot | Bin 0 -> 25395 bytes .../static/fonts/fontawesome-webfont.svg | 284 + .../static/fonts/fontawesome-webfont.ttf | Bin 0 -> 55096 bytes .../static/fonts/fontawesome-webfont.woff | Bin 0 -> 29380 bytes .../static/fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20127 bytes .../static/fonts/glyphicons-halflings-regular.svg | 288 + .../static/fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 45404 bytes .../static/fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23424 bytes .../fonts/glyphicons-halflings-regular.woff2 | Bin 0 -> 18028 bytes .../toastergui/static/html/layer_deps_modal.html | 21 + .../lib/toaster/toastergui/static/img/logo.png | Bin 0 -> 3020 bytes .../toaster/toastergui/static/img/toaster_bw.png | Bin 0 -> 140514 bytes .../jquery-treetable-license/GPL-LICENSE.txt | 278 + .../jquery-treetable-license/MIT-LICENSE.txt | 20 + .../static/jquery-treetable-license/README.md | 20 + .../static/jquery.treetable.theme.toaster.css | 66 + .../lib/toaster/toastergui/static/js/.jshintrc | 11 + .../lib/toaster/toastergui/static/js/bootstrap.js | 2363 +++++++ .../toaster/toastergui/static/js/bootstrap.min.js | 7 + .../toaster/toastergui/static/js/customrecipe.js | 322 + .../toaster/toastergui/static/js/filtersnippet.js | 94 + .../toaster/toastergui/static/js/highlight.pack.js | 2 + .../toaster/toastergui/static/js/importlayer.js | 447 ++ .../toastergui/static/js/jquery-2.0.3.min.js | 6 + .../toastergui/static/js/jquery-2.0.3.min.map | 1 + .../toaster/toastergui/static/js/jquery-ui.min.js | 7 + .../toaster/toastergui/static/js/jquery.cookie.js | 117 + .../toastergui/static/js/jquery.treetable.js | 620 ++ .../toaster/toastergui/static/js/jsrender.min.js | 4 + .../lib/toaster/toastergui/static/js/layerBtn.js | 83 + .../toaster/toastergui/static/js/layerDepsModal.js | 98 + .../toaster/toastergui/static/js/layerdetails.js | 521 ++ .../lib/toaster/toastergui/static/js/libtoaster.js | 735 +++ .../lib/toaster/toastergui/static/js/mrbsection.js | 150 + .../toastergui/static/js/newcustomimage_modal.js | 199 + .../toaster/toastergui/static/js/projectpage.js | 390 ++ .../toaster/toastergui/static/js/projecttopbar.js | 97 + .../toaster/toastergui/static/js/qunit-1.18.0.js | 347 + .../toaster/toastergui/static/js/recipedetails.js | 51 + .../lib/toaster/toastergui/static/js/table.js | 857 +++ .../lib/toaster/toastergui/static/js/tests/test.js | 177 + .../toastergui/static/js/typeahead.jquery.js | 1551 +++++ .../static/js/ui-bootstrap-tpls-0.11.0.js | 10 + .../static/js/ui-bootstrap-tpls-0.11.0.min.js | 10 + poky/bitbake/lib/toaster/toastergui/tablefilter.py | 292 + poky/bitbake/lib/toaster/toastergui/tables.py | 1629 +++++ .../lib/toaster/toastergui/templates/base.html | 144 + .../toastergui/templates/basebuilddetailpage.html | 31 + .../toastergui/templates/basebuildpage.html | 234 + .../templates/baseprojectbuildspage.html | 15 + .../toastergui/templates/baseprojectpage.html | 52 + .../toastergui/templates/basetable_bottom.html | 94 + .../toastergui/templates/basetable_top.html | 256 + .../toastergui/templates/basetable_top_layers.html | 5 + .../lib/toaster/toastergui/templates/bfile.html | 24 + .../toaster/toastergui/templates/brtargets.html | 20 + .../toastergui/templates/builddashboard.html | 319 + .../templates/buildinfo-toastertable.html | 25 + .../toastergui/templates/builds-toastertable.html | 52 + .../toastergui/templates/configuration.html | 85 + .../toaster/toastergui/templates/configvars.html | 145 + .../lib/toaster/toastergui/templates/cpuusage.html | 4 + .../toastergui/templates/customise_btn.html | 15 + .../toaster/toastergui/templates/customrecipe.html | 268 + .../templates/detail_pagination_bottom.html | 57 + .../toastergui/templates/detail_search_header.html | 72 + .../toastergui/templates/detail_sorted_header.html | 25 + .../lib/toaster/toastergui/templates/dirinfo.html | 245 + .../lib/toaster/toastergui/templates/diskio.html | 4 + .../toaster/toastergui/templates/distro_btn.html | 20 + .../templates/editcustomimage_modal.html | 88 + .../toastergui/templates/filtersnippet.html | 72 + .../templates/generic-toastertable-page.html | 17 + .../lib/toaster/toastergui/templates/health.html | 6 + .../toaster/toastergui/templates/importlayer.html | 170 + .../toastergui/templates/js-unit-tests.html | 47 + .../lib/toaster/toastergui/templates/landing.html | 67 + .../toastergui/templates/landing_not_managed.html | 34 + .../toaster/toastergui/templates/layer_btn.html | 26 + .../toaster/toastergui/templates/layerdetails.html | 373 ++ .../toaster/toastergui/templates/machine_btn.html | 21 + .../toaster/toastergui/templates/mrb_section.html | 301 + .../toastergui/templates/newcustomimage.html | 21 + .../toastergui/templates/newcustomimage_modal.html | 60 + .../toaster/toastergui/templates/newproject.html | 138 + .../templates/package_built_dependencies.html | 99 + .../toastergui/templates/package_built_detail.html | 65 + .../toastergui/templates/package_detail_base.html | 169 + .../templates/package_included_dependencies.html | 110 + .../templates/package_included_detail.html | 44 + .../package_included_reverse_dependencies.html | 50 + .../templates/package_included_tabs.html | 33 + .../toastergui/templates/pkg_add_rm_btn.html | 34 + .../lib/toaster/toastergui/templates/project.html | 161 + .../templates/projectbuilds-toastertable.html | 65 + .../toastergui/templates/projectbuilds.html | 118 + .../toaster/toastergui/templates/projectconf.html | 1042 +++ .../templates/projects-toastertable.html | 42 + .../toastergui/templates/projecttopbar.html | 79 + .../lib/toaster/toastergui/templates/recipe.html | 320 + .../toaster/toastergui/templates/recipe_btn.html | 23 + .../toastergui/templates/recipe_packages.html | 135 + .../toastergui/templates/recipedetails.html | 175 + .../templates/snippets/gitrev_popover.html | 8 + .../snippets/pkg_dependencies_popover.html | 38 + .../snippets/pkg_revdependencies_popover.html | 38 + .../toaster/toastergui/templates/tablesort.html | 38 + .../lib/toaster/toastergui/templates/target.html | 56 + .../lib/toaster/toastergui/templates/task.html | 348 + .../toastergui/templates/toastertable-filter.html | 24 + .../toastergui/templates/toastertable-simple.html | 100 + .../toaster/toastergui/templates/toastertable.html | 125 + .../toastergui/templates/unavailable_artifact.html | 18 + .../toaster/toastergui/templatetags/__init__.py | 0 .../toastergui/templatetags/field_values_filter.py | 18 + .../templatetags/objects_to_dictionaries_filter.py | 35 + .../toastergui/templatetags/project_url_tag.py | 34 + .../toaster/toastergui/templatetags/projecttags.py | 299 + poky/bitbake/lib/toaster/toastergui/typeaheads.py | 211 + poky/bitbake/lib/toaster/toastergui/urls.py | 254 + poky/bitbake/lib/toaster/toastergui/views.py | 1791 ++++++ poky/bitbake/lib/toaster/toastergui/widgets.py | 577 ++ poky/bitbake/lib/toaster/toastermain/__init__.py | 0 .../lib/toaster/toastermain/management/__init__.py | 0 .../toastermain/management/commands/__init__.py | 0 .../toastermain/management/commands/builddelete.py | 54 + .../toastermain/management/commands/buildslist.py | 13 + .../toastermain/management/commands/checksocket.py | 69 + .../toastermain/management/commands/perf.py | 58 + poky/bitbake/lib/toaster/toastermain/settings.py | 369 ++ .../toastermain/settings_production_example.py | 58 + .../lib/toaster/toastermain/settings_test.py | 41 + poky/bitbake/lib/toaster/toastermain/urls.py | 93 + poky/bitbake/lib/toaster/toastermain/wsgi.py | 35 + 263 files changed, 43204 insertions(+) create mode 100644 poky/bitbake/lib/toaster/__init__.py create mode 100644 poky/bitbake/lib/toaster/bldcollector/__init__.py create mode 100644 poky/bitbake/lib/toaster/bldcollector/admin.py create mode 100644 poky/bitbake/lib/toaster/bldcollector/urls.py create mode 100644 poky/bitbake/lib/toaster/bldcollector/views.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/__init__.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/admin.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/management/__init__.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/migrations/__init__.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/models.py create mode 100644 poky/bitbake/lib/toaster/bldcontrol/views.py create mode 100755 poky/bitbake/lib/toaster/manage.py create mode 100644 poky/bitbake/lib/toaster/orm/__init__.py create mode 100644 poky/bitbake/lib/toaster/orm/fixtures/README create mode 100755 poky/bitbake/lib/toaster/orm/fixtures/custom_toaster_append.sh_sample create mode 100644 poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml create mode 100644 poky/bitbake/lib/toaster/orm/fixtures/poky.xml create mode 100644 poky/bitbake/lib/toaster/orm/fixtures/settings.xml create mode 100644 poky/bitbake/lib/toaster/orm/management/__init__.py create mode 100644 poky/bitbake/lib/toaster/orm/management/commands/__init__.py create mode 100644 poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0001_initial.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0002_customimagerecipe.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0003_customimagepackage.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0004_provides.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0006_add_cancelled_state.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0007_auto_20160523_1446.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0009_target_package_manifest_path.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0010_delete_layer_source_references.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0011_delete_layersource.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0012_use_release_instead_of_up_branch.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0015_layer_local_source_dir.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0016_clone_progress.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/0017_distro_clone.py create mode 100644 poky/bitbake/lib/toaster/orm/migrations/__init__.py create mode 100644 poky/bitbake/lib/toaster/orm/models.py create mode 100644 poky/bitbake/lib/toaster/tests/__init__.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/README create mode 100644 poky/bitbake/lib/toaster/tests/browser/__init__.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/selenium_helpers.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_all_builds_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_all_projects_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_builddashboard_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_builddashboard_page_recipes.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_builddashboard_page_tasks.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_js_unit_tests.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_landing_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_new_project_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_project_builds_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_project_config_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_project_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_sample.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_task_page.py create mode 100644 poky/bitbake/lib/toaster/tests/browser/test_toastertable_ui.py create mode 100644 poky/bitbake/lib/toaster/tests/builds/README create mode 100644 poky/bitbake/lib/toaster/tests/builds/__init__.py create mode 100644 poky/bitbake/lib/toaster/tests/builds/buildtest.py create mode 100644 poky/bitbake/lib/toaster/tests/builds/test_core_image_min.py create mode 100644 poky/bitbake/lib/toaster/tests/commands/__init__.py create mode 100644 poky/bitbake/lib/toaster/tests/commands/test_loaddata.py create mode 100644 poky/bitbake/lib/toaster/tests/commands/test_lsupdates.py create mode 100644 poky/bitbake/lib/toaster/tests/commands/test_runbuilds.py create mode 100644 poky/bitbake/lib/toaster/tests/db/__init__.py create mode 100644 poky/bitbake/lib/toaster/tests/db/test_db.py create mode 100644 poky/bitbake/lib/toaster/tests/eventreplay/README create mode 100644 poky/bitbake/lib/toaster/tests/eventreplay/__init__.py create mode 100644 poky/bitbake/lib/toaster/tests/functional/README create mode 100644 poky/bitbake/lib/toaster/tests/functional/__init__.py create mode 100644 poky/bitbake/lib/toaster/tests/functional/functional_helpers.py create mode 100644 poky/bitbake/lib/toaster/tests/functional/test_functional_basic.py create mode 100644 poky/bitbake/lib/toaster/tests/toaster-tests-requirements.txt create mode 100644 poky/bitbake/lib/toaster/tests/views/README create mode 100644 poky/bitbake/lib/toaster/tests/views/__init__.py create mode 100644 poky/bitbake/lib/toaster/tests/views/test_views.py create mode 100644 poky/bitbake/lib/toaster/toastergui/__init__.py create mode 100644 poky/bitbake/lib/toaster/toastergui/api.py create mode 100644 poky/bitbake/lib/toaster/toastergui/buildtables.py create mode 100644 poky/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/bootstrap.css create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/bootstrap.css.map create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/bootstrap.min.css create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/bootstrap.min.css.map create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/bootstrap3-transition.css create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/default.css create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/font-awesome.min.css create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_diagonals-thick_18_b81900_40x40.png create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_diagonals-thick_20_666666_40x40.png create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_flat_10_000000_40x100.png create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_glass_100_f6f6f6_1x400.png create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_glass_100_fdf5ce_1x400.png create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_glass_65_ffffff_1x400.png create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_gloss-wave_35_f6a828_500x100.png create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_highlight-soft_100_eeeeee_1x100.png create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-bg_highlight-soft_75_ffe45c_1x100.png create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-icons_222222_256x240.png create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-icons_228ef1_256x240.png create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-icons_ef8c08_256x240.png create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-icons_ffd27a_256x240.png create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/images/ui-icons_ffffff_256x240.png create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/jquery-ui.min.css create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/jquery-ui.structure.min.css create mode 100755 poky/bitbake/lib/toaster/toastergui/static/css/jquery-ui.theme.min.css create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/jquery.treetable.css create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/jquery.treetable.theme.default.css create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/jquery.treetable.theme.toaster.css create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/qunit-1.18.0.css create mode 100644 poky/bitbake/lib/toaster/toastergui/static/css/screen.css create mode 100644 poky/bitbake/lib/toaster/toastergui/static/fonts/FontAwesome.otf create mode 100644 poky/bitbake/lib/toaster/toastergui/static/fonts/fontawesome-webfont.eot create mode 100644 poky/bitbake/lib/toaster/toastergui/static/fonts/fontawesome-webfont.svg create mode 100644 poky/bitbake/lib/toaster/toastergui/static/fonts/fontawesome-webfont.ttf create mode 100644 poky/bitbake/lib/toaster/toastergui/static/fonts/fontawesome-webfont.woff create mode 100644 poky/bitbake/lib/toaster/toastergui/static/fonts/glyphicons-halflings-regular.eot create mode 100644 poky/bitbake/lib/toaster/toastergui/static/fonts/glyphicons-halflings-regular.svg create mode 100644 poky/bitbake/lib/toaster/toastergui/static/fonts/glyphicons-halflings-regular.ttf create mode 100644 poky/bitbake/lib/toaster/toastergui/static/fonts/glyphicons-halflings-regular.woff create mode 100644 poky/bitbake/lib/toaster/toastergui/static/fonts/glyphicons-halflings-regular.woff2 create mode 100644 poky/bitbake/lib/toaster/toastergui/static/html/layer_deps_modal.html create mode 100644 poky/bitbake/lib/toaster/toastergui/static/img/logo.png create mode 100644 poky/bitbake/lib/toaster/toastergui/static/img/toaster_bw.png create mode 100644 poky/bitbake/lib/toaster/toastergui/static/jquery-treetable-license/GPL-LICENSE.txt create mode 100644 poky/bitbake/lib/toaster/toastergui/static/jquery-treetable-license/MIT-LICENSE.txt create mode 100644 poky/bitbake/lib/toaster/toastergui/static/jquery-treetable-license/README.md create mode 100644 poky/bitbake/lib/toaster/toastergui/static/jquery.treetable.theme.toaster.css create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/.jshintrc create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/bootstrap.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/bootstrap.min.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/customrecipe.js create mode 100755 poky/bitbake/lib/toaster/toastergui/static/js/filtersnippet.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/highlight.pack.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/importlayer.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/jquery-2.0.3.min.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/jquery-2.0.3.min.map create mode 100755 poky/bitbake/lib/toaster/toastergui/static/js/jquery-ui.min.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/jquery.cookie.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/jquery.treetable.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/jsrender.min.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/layerBtn.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/layerDepsModal.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/layerdetails.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/libtoaster.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/mrbsection.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/newcustomimage_modal.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/projectpage.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/projecttopbar.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/qunit-1.18.0.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/recipedetails.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/table.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/tests/test.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/typeahead.jquery.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/ui-bootstrap-tpls-0.11.0.js create mode 100644 poky/bitbake/lib/toaster/toastergui/static/js/ui-bootstrap-tpls-0.11.0.min.js create mode 100644 poky/bitbake/lib/toaster/toastergui/tablefilter.py create mode 100644 poky/bitbake/lib/toaster/toastergui/tables.py create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/base.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/basebuilddetailpage.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/basebuildpage.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/baseprojectbuildspage.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/baseprojectpage.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/basetable_bottom.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/basetable_top.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/basetable_top_layers.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/bfile.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/brtargets.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/builddashboard.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/buildinfo-toastertable.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/builds-toastertable.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/configuration.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/configvars.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/cpuusage.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/customise_btn.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/customrecipe.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/detail_pagination_bottom.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/detail_search_header.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/detail_sorted_header.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/dirinfo.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/diskio.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/distro_btn.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/editcustomimage_modal.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/filtersnippet.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/generic-toastertable-page.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/health.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/importlayer.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/js-unit-tests.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/landing.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/landing_not_managed.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/layer_btn.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/layerdetails.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/machine_btn.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/mrb_section.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/newcustomimage.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/newcustomimage_modal.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/newproject.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/package_built_dependencies.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/package_built_detail.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/package_detail_base.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/package_included_dependencies.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/package_included_detail.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/package_included_reverse_dependencies.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/package_included_tabs.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/pkg_add_rm_btn.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/project.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/projectbuilds-toastertable.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/projectconf.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/projects-toastertable.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/projecttopbar.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/recipe.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/recipe_btn.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/recipe_packages.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/recipedetails.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/snippets/gitrev_popover.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/snippets/pkg_dependencies_popover.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/snippets/pkg_revdependencies_popover.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/tablesort.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/target.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/task.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/toastertable-filter.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/toastertable-simple.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/toastertable.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templates/unavailable_artifact.html create mode 100644 poky/bitbake/lib/toaster/toastergui/templatetags/__init__.py create mode 100644 poky/bitbake/lib/toaster/toastergui/templatetags/field_values_filter.py create mode 100644 poky/bitbake/lib/toaster/toastergui/templatetags/objects_to_dictionaries_filter.py create mode 100644 poky/bitbake/lib/toaster/toastergui/templatetags/project_url_tag.py create mode 100644 poky/bitbake/lib/toaster/toastergui/templatetags/projecttags.py create mode 100644 poky/bitbake/lib/toaster/toastergui/typeaheads.py create mode 100644 poky/bitbake/lib/toaster/toastergui/urls.py create mode 100755 poky/bitbake/lib/toaster/toastergui/views.py create mode 100644 poky/bitbake/lib/toaster/toastergui/widgets.py create mode 100644 poky/bitbake/lib/toaster/toastermain/__init__.py create mode 100644 poky/bitbake/lib/toaster/toastermain/management/__init__.py create mode 100644 poky/bitbake/lib/toaster/toastermain/management/commands/__init__.py create mode 100644 poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py create mode 100644 poky/bitbake/lib/toaster/toastermain/management/commands/buildslist.py create mode 100644 poky/bitbake/lib/toaster/toastermain/management/commands/checksocket.py create mode 100644 poky/bitbake/lib/toaster/toastermain/management/commands/perf.py create mode 100644 poky/bitbake/lib/toaster/toastermain/settings.py create mode 100644 poky/bitbake/lib/toaster/toastermain/settings_production_example.py create mode 100644 poky/bitbake/lib/toaster/toastermain/settings_test.py create mode 100644 poky/bitbake/lib/toaster/toastermain/urls.py create mode 100644 poky/bitbake/lib/toaster/toastermain/wsgi.py (limited to 'poky/bitbake/lib/toaster') diff --git a/poky/bitbake/lib/toaster/__init__.py b/poky/bitbake/lib/toaster/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/bldcollector/__init__.py b/poky/bitbake/lib/toaster/bldcollector/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/bldcollector/admin.py b/poky/bitbake/lib/toaster/bldcollector/admin.py new file mode 100644 index 000000000..1f2e07f50 --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcollector/admin.py @@ -0,0 +1,33 @@ +from django.contrib import admin +from orm.models import BitbakeVersion, Release, ToasterSetting, Layer_Version +from django import forms +import django.db.models as models + + +class BitbakeVersionAdmin(admin.ModelAdmin): + + # we override the formfield for db URLField + # because of broken URL validation + + def formfield_for_dbfield(self, db_field, **kwargs): + if isinstance(db_field, models.fields.URLField): + return forms.fields.CharField() + return super(BitbakeVersionAdmin, self).formfield_for_dbfield( + db_field, **kwargs) + + +class ReleaseAdmin(admin.ModelAdmin): + pass + + +class ToasterSettingAdmin(admin.ModelAdmin): + pass + + +class LayerVersionsAdmin(admin.ModelAdmin): + pass + +admin.site.register(Layer_Version, LayerVersionsAdmin) +admin.site.register(BitbakeVersion, BitbakeVersionAdmin) +admin.site.register(Release, ReleaseAdmin) +admin.site.register(ToasterSetting, ToasterSettingAdmin) diff --git a/poky/bitbake/lib/toaster/bldcollector/urls.py b/poky/bitbake/lib/toaster/bldcollector/urls.py new file mode 100644 index 000000000..888175d0a --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcollector/urls.py @@ -0,0 +1,27 @@ +# +# BitBake Toaster Implementation +# +# Copyright (C) 2014-2017 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from django.conf.urls import include, url + +import bldcollector.views + +urlpatterns = [ + # landing point for pushing a bitbake_eventlog.json file to this toaster instace + url(r'^eventfile$', bldcollector.views.eventfile, name='eventfile'), +] diff --git a/poky/bitbake/lib/toaster/bldcollector/views.py b/poky/bitbake/lib/toaster/bldcollector/views.py new file mode 100644 index 000000000..f32fa4d22 --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcollector/views.py @@ -0,0 +1,62 @@ +# +# BitBake Toaster Implementation +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from django.views.decorators.cache import cache_control +from django.core.urlresolvers import reverse +from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.http import HttpResponseBadRequest, HttpResponse +from django.utils import timezone +from django.utils.html import escape +from datetime import timedelta +from django.utils import formats +from toastergui.templatetags.projecttags import json as jsonfilter +import json +import os +import tempfile +import subprocess +import toastermain +from django.views.decorators.csrf import csrf_exempt + + +@csrf_exempt +def eventfile(request): + """ Receives a file by POST, and runs toaster-eventreply on this file """ + if request.method != "POST": + return HttpResponseBadRequest("This API only accepts POST requests. Post a file with:\n\ncurl -F eventlog=@bitbake_eventlog.json %s\n" % request.build_absolute_uri(reverse('eventfile')), content_type="text/plain;utf8") + + # write temporary file + (handle, abstemppath) = tempfile.mkstemp(dir="/tmp/") + with os.fdopen(handle, "w") as tmpfile: + for chunk in request.FILES['eventlog'].chunks(): + tmpfile.write(chunk) + tmpfile.close() + + # compute the path to "bitbake/bin/toaster-eventreplay" + from os.path import dirname as DN + import_script = os.path.join(DN(DN(DN(DN(os.path.abspath(__file__))))), "bin/toaster-eventreplay") + if not os.path.exists(import_script): + raise Exception("script missing %s" % import_script) + scriptenv = os.environ.copy() + scriptenv["DATABASE_URL"] = toastermain.settings.getDATABASE_URL() + + # run the data loading process and return the results + importer = subprocess.Popen([import_script, abstemppath], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=scriptenv) + (out, err) = importer.communicate() + if importer.returncode == 0: + os.remove(abstemppath) + return HttpResponse("== Retval %d\n== STDOUT\n%s\n\n== STDERR\n%s" % (importer.returncode, out, err), content_type="text/plain;utf8") diff --git a/poky/bitbake/lib/toaster/bldcontrol/__init__.py b/poky/bitbake/lib/toaster/bldcontrol/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/bldcontrol/admin.py b/poky/bitbake/lib/toaster/bldcontrol/admin.py new file mode 100644 index 000000000..fcbe5f593 --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/admin.py @@ -0,0 +1,8 @@ +from django.contrib import admin +from django.contrib.admin.filters import RelatedFieldListFilter +from .models import BuildEnvironment + +class BuildEnvironmentAdmin(admin.ModelAdmin): + pass + +admin.site.register(BuildEnvironment, BuildEnvironmentAdmin) diff --git a/poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py new file mode 100644 index 000000000..5195600d9 --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -0,0 +1,142 @@ +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import sys +import re +from django.db import transaction +from django.db.models import Q +from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake + +# load Bitbake components +path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +sys.path.insert(0, path) + +class BitbakeController(object): + """ This is the basic class that controlls a bitbake server. + It is outside the scope of this class on how the server is started and aquired + """ + + def __init__(self, be): + import bb.server.xmlrpcclient + self.connection = bb.server.xmlrpcclient._create_server(be.bbaddress, + int(be.bbport))[0] + + def _runCommand(self, command): + result, error = self.connection.runCommand(command) + if error: + raise Exception(error) + return result + + def disconnect(self): + return self.connection.removeClient() + + def setVariable(self, name, value): + return self._runCommand(["setVariable", name, value]) + + def getVariable(self, name): + return self._runCommand(["getVariable", name]) + + def triggerEvent(self, event): + return self._runCommand(["triggerEvent", event]) + + def build(self, targets, task = None): + if task is None: + task = "build" + return self._runCommand(["buildTargets", targets, task]) + + def forceShutDown(self): + return self._runCommand(["stateForceShutdown"]) + + + +def getBuildEnvironmentController(**kwargs): + """ Gets you a BuildEnvironmentController that encapsulates a build environment, + based on the query dictionary sent in. + + This is used to retrieve, for example, the currently running BE from inside + the toaster UI, or find a new BE to start a new build in it. + + The return object MUST always be a BuildEnvironmentController. + """ + + from bldcontrol.localhostbecontroller import LocalhostBEController + + be = BuildEnvironment.objects.filter(Q(**kwargs))[0] + if be.betype == BuildEnvironment.TYPE_LOCAL: + return LocalhostBEController(be) + else: + raise Exception("FIXME: Implement BEC for type %s" % str(be.betype)) + + +class BuildEnvironmentController(object): + """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST + or SHOULD be supported by a Build Environment. It is used to establish the framework, and must + not be instantiated directly by the user. + + Use the "getBuildEnvironmentController()" function to get a working BEC for your remote. + + How the BuildEnvironments are discovered is outside the scope of this class. + + You must derive this class to teach Toaster how to operate in your own infrastructure. + We provide some specific BuildEnvironmentController classes that can be used either to + directly set-up Toaster infrastructure, or as a model for your own infrastructure set: + + * Localhost controller will run the Toaster BE on the same account as the web server + (current user if you are using the the Django development web server) + on the local machine, with the "build/" directory under the "poky/" source checkout directory. + Bash is expected to be available. + + """ + def __init__(self, be): + """ Takes a BuildEnvironment object as parameter that points to the settings of the BE. + """ + self.be = be + self.connection = None + + def setLayers(self, bitbake, ls): + """ Checks-out bitbake executor and layers from git repositories. + Sets the layer variables in the config file, after validating local layer paths. + bitbake must be a single BRBitbake instance + The layer paths must be in a list of BRLayer object + + a word of attention: by convention, the first layer for any build will be poky! + """ + raise NotImplementedError("FIXME: Must override setLayers") + + def getArtifact(self, path): + """ This call returns an artifact identified by the 'path'. How 'path' is interpreted as + up to the implementing BEC. The return MUST be a REST URL where a GET will actually return + the content of the artifact, e.g. for use as a "download link" in a web UI. + """ + raise NotImplementedError("Must return the REST URL of the artifact") + + def triggerBuild(self, bitbake, layers, variables, targets): + raise NotImplementedError("Must override BE release") + +class ShellCmdException(Exception): + pass + + +class BuildSetupException(Exception): + pass + diff --git a/poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py new file mode 100644 index 000000000..16c7c8044 --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -0,0 +1,401 @@ +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import sys +import re +import shutil +import time +from django.db import transaction +from django.db.models import Q +from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake +from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer, ToasterSetting +import subprocess + +from toastermain import settings + +from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, BitbakeController + +import logging +logger = logging.getLogger("toaster") + +from pprint import pprint, pformat + +class LocalhostBEController(BuildEnvironmentController): + """ Implementation of the BuildEnvironmentController for the localhost; + this controller manages the default build directory, + the server setup and system start and stop for the localhost-type build environment + + """ + + def __init__(self, be): + super(LocalhostBEController, self).__init__(be) + self.pokydirname = None + self.islayerset = False + + def _shellcmd(self, command, cwd=None, nowait=False,env=None): + if cwd is None: + cwd = self.be.sourcedir + if env is None: + env=os.environ.copy() + + logger.debug("lbc_shellcmd: (%s) %s" % (cwd, command)) + p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + if nowait: + return + (out,err) = p.communicate() + p.wait() + if p.returncode: + if len(err) == 0: + err = "command: %s \n%s" % (command, out) + else: + err = "command: %s \n%s" % (command, err) + logger.warning("localhostbecontroller: shellcmd error %s" % err) + raise ShellCmdException(err) + else: + logger.debug("localhostbecontroller: shellcmd success") + return out.decode('utf-8') + + def getGitCloneDirectory(self, url, branch): + """Construct unique clone directory name out of url and branch.""" + if branch != "HEAD": + return "_toaster_clones/_%s_%s" % (re.sub('[:/@+%]', '_', url), branch) + + # word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases + # which _ALWAYS_ means the current poky checkout + from os.path import dirname as DN + local_checkout_path = DN(DN(DN(DN(DN(os.path.abspath(__file__)))))) + #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path) + return local_checkout_path + + + def setCloneStatus(self,bitbake,status,total,current): + bitbake.req.build.repos_cloned=current + bitbake.req.build.repos_to_clone=total + bitbake.req.build.save() + + def setLayers(self, bitbake, layers, targets): + """ a word of attention: by convention, the first layer for any build will be poky! """ + + assert self.be.sourcedir is not None + + layerlist = [] + nongitlayerlist = [] + git_env = os.environ.copy() + # (note: add custom environment settings here) + + # set layers in the layersource + + # 1. get a list of repos with branches, and map dirpaths for each layer + gitrepos = {} + + # if we're using a remotely fetched version of bitbake add its git + # details to the list of repos to clone + if bitbake.giturl and bitbake.commit: + gitrepos[(bitbake.giturl, bitbake.commit)] = [] + gitrepos[(bitbake.giturl, bitbake.commit)].append( + ("bitbake", bitbake.dirpath)) + + for layer in layers: + # We don't need to git clone the layer for the CustomImageRecipe + # as it's generated by us layer on if needed + if CustomImageRecipe.LAYER_NAME in layer.name: + continue + + # If we have local layers then we don't need clone them + # For local layers giturl will be empty + if not layer.giturl: + nongitlayerlist.append(layer.layer_version.layer.local_source_dir) + continue + + if not (layer.giturl, layer.commit) in gitrepos: + gitrepos[(layer.giturl, layer.commit)] = [] + gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) + + + logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) + + + # 2. Note for future use if the current source directory is a + # checked-out git repos that could match a layer's vcs_url and therefore + # be used to speed up cloning (rather than fetching it again). + + cached_layers = {} + + try: + for remotes in self._shellcmd("git remote -v", self.be.sourcedir,env=git_env).split("\n"): + try: + remote = remotes.split("\t")[1].split(" ")[0] + if remote not in cached_layers: + cached_layers[remote] = self.be.sourcedir + except IndexError: + pass + except ShellCmdException: + # ignore any errors in collecting git remotes this is an optional + # step + pass + + logger.info("Using pre-checked out source for layer %s", cached_layers) + + # 3. checkout the repositories + clone_count=0 + clone_total=len(gitrepos.keys()) + self.setCloneStatus(bitbake,'Started',clone_total,clone_count) + for giturl, commit in gitrepos.keys(): + self.setCloneStatus(bitbake,'progress',clone_total,clone_count) + clone_count += 1 + + localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) + logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) + + # see if our directory is a git repository + if os.path.exists(localdirname): + try: + localremotes = self._shellcmd("git remote -v", + localdirname,env=git_env) + if not giturl in localremotes and commit != 'HEAD': + raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) + except ShellCmdException: + # our localdirname might not be a git repository + #- that's fine + pass + else: + if giturl in cached_layers: + logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) + self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname),env=git_env) + self._shellcmd("git remote remove origin", localdirname,env=git_env) + self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname,env=git_env) + else: + logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname)) + self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname),env=git_env) + + # branch magic name "HEAD" will inhibit checkout + if commit != "HEAD": + logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) + ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit + self._shellcmd('git fetch && git reset --hard "%s"' % ref, localdirname,env=git_env) + + # take the localdirname as poky dir if we can find the oe-init-build-env + if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): + logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname) + self.pokydirname = localdirname + + # make sure we have a working bitbake + if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): + logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env) + + # verify our repositories + for name, dirpath in gitrepos[(giturl, commit)]: + localdirpath = os.path.join(localdirname, dirpath) + logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) + if not os.path.exists(localdirpath): + raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) + + if name != "bitbake": + layerlist.append(localdirpath.rstrip("/")) + + self.setCloneStatus(bitbake,'complete',clone_total,clone_count) + logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) + + if self.pokydirname is None and os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")): + logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir) + self.pokydirname = self.be.sourcedir + + # 5. create custom layer and add custom recipes to it + for target in targets: + try: + customrecipe = CustomImageRecipe.objects.get( + name=target.target, + project=bitbake.req.project) + + custom_layer_path = self.setup_custom_image_recipe( + customrecipe, layers) + + if os.path.isdir(custom_layer_path): + layerlist.append(custom_layer_path) + + except CustomImageRecipe.DoesNotExist: + continue # not a custom recipe, skip + + layerlist.extend(nongitlayerlist) + logger.debug("\n\nset layers gives this list %s" % pformat(layerlist)) + self.islayerset = True + return layerlist + + def setup_custom_image_recipe(self, customrecipe, layers): + """ Set up toaster-custom-images layer and recipe files """ + layerpath = os.path.join(self.be.builddir, + CustomImageRecipe.LAYER_NAME) + + # create directory structure + for name in ("conf", "recipes"): + path = os.path.join(layerpath, name) + if not os.path.isdir(path): + os.makedirs(path) + + # create layer.conf + config = os.path.join(layerpath, "conf", "layer.conf") + if not os.path.isfile(config): + with open(config, "w") as conf: + conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n') + + # Update the Layer_Version dirpath that has our base_recipe in + # to be able to read the base recipe to then generate the + # custom recipe. + br_layer_base_recipe = layers.get( + layer_version=customrecipe.base_recipe.layer_version) + + # If the layer is one that we've cloned we know where it lives + if br_layer_base_recipe.giturl and br_layer_base_recipe.commit: + layer_path = self.getGitCloneDirectory( + br_layer_base_recipe.giturl, + br_layer_base_recipe.commit) + # Otherwise it's a local layer + elif br_layer_base_recipe.local_source_dir: + layer_path = br_layer_base_recipe.local_source_dir + else: + logger.error("Unable to workout the dir path for the custom" + " image recipe") + + br_layer_base_dirpath = os.path.join( + self.be.sourcedir, + layer_path, + customrecipe.base_recipe.layer_version.dirpath) + + customrecipe.base_recipe.layer_version.dirpath = br_layer_base_dirpath + + customrecipe.base_recipe.layer_version.save() + + # create recipe + recipe_path = os.path.join(layerpath, "recipes", "%s.bb" % + customrecipe.name) + with open(recipe_path, "w") as recipef: + recipef.write(customrecipe.generate_recipe_file_contents()) + + # Update the layer and recipe objects + customrecipe.layer_version.dirpath = layerpath + customrecipe.layer_version.layer.local_source_dir = layerpath + customrecipe.layer_version.layer.save() + customrecipe.layer_version.save() + + customrecipe.file_path = recipe_path + customrecipe.save() + + return layerpath + + + def readServerLogFile(self): + return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read() + + + def triggerBuild(self, bitbake, layers, variables, targets, brbe): + layers = self.setLayers(bitbake, layers, targets) + + # init build environment from the clone + builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) + oe_init = os.path.join(self.pokydirname, 'oe-init-build-env') + # init build environment + try: + custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value + custom_script = custom_script.replace("%BUILDDIR%" ,builddir) + self._shellcmd("bash -c 'source %s'" % (custom_script)) + except ToasterSetting.DoesNotExist: + self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir), + self.be.sourcedir) + + # update bblayers.conf + bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf") + with open(bblconfpath, 'w') as bblayers: + bblayers.write('# line added by toaster build control\n' + 'BBLAYERS = "%s"' % ' '.join(layers)) + + # write configuration file + confpath = os.path.join(builddir, 'conf/toaster.conf') + with open(confpath, 'w') as conf: + for var in variables: + conf.write('%s="%s"\n' % (var.name, var.value)) + conf.write('INHERIT+="toaster buildhistory"') + + # clean the Toaster to build environment + env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0 + + # run bitbake server from the clone + bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake') + toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf") + self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s ' + '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, + builddir, bitbake, confpath, toasterlayers), self.be.sourcedir) + + # read port number from bitbake.lock + self.be.bbport = -1 + bblock = os.path.join(builddir, 'bitbake.lock') + # allow 10 seconds for bb lock file to appear but also be populated + for lock_check in range(10): + if not os.path.exists(bblock): + logger.debug("localhostbecontroller: waiting for bblock file to appear") + time.sleep(1) + continue + if 10 < os.stat(bblock).st_size: + break + logger.debug("localhostbecontroller: waiting for bblock content to appear") + time.sleep(1) + else: + raise BuildSetupException("Cannot find bitbake server lock file '%s'. Aborting." % bblock) + + with open(bblock) as fplock: + for line in fplock: + if ":" in line: + self.be.bbport = line.split(":")[-1].strip() + logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport) + break + + if -1 == self.be.bbport: + raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock) + + self.be.bbaddress = "localhost" + self.be.bbstate = BuildEnvironment.SERVER_STARTED + self.be.lock = BuildEnvironment.LOCK_RUNNING + self.be.save() + + bbtargets = '' + for target in targets: + task = target.task + if task: + if not task.startswith('do_'): + task = 'do_' + task + task = ':%s' % task + bbtargets += '%s%s ' % (target.target, task) + + # run build with local bitbake. stop the server after the build. + log = os.path.join(builddir, 'toaster_ui.log') + local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')), + 'bitbake') + self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' + '%s %s -u toasterui --read %s --read %s --token="" >>%s 2>&1;' + 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ + % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log, + self.be.bbport, bitbake,)], + builddir, nowait=True) + + logger.debug('localhostbecontroller: Build launched, exiting. ' + 'Follow build logs at %s' % log) diff --git a/poky/bitbake/lib/toaster/bldcontrol/management/__init__.py b/poky/bitbake/lib/toaster/bldcontrol/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py b/poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py new file mode 100644 index 000000000..823c6f154 --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -0,0 +1,167 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from django.core.management import call_command +from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError +from orm.models import ToasterSetting, Build, Layer + +import os +import traceback +import warnings + + +def DN(path): + if path is None: + return "" + else: + return os.path.dirname(path) + + +class Command(BaseCommand): + args = "" + help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings." + + def __init__(self, *args, **kwargs): + super(Command, self).__init__(*args, **kwargs) + self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__))))))) + + def _verify_build_environment(self): + # provide a local build env. This will be extended later to include non local + if BuildEnvironment.objects.count() == 0: + BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL) + + # we make sure we have builddir and sourcedir for all defined build envionments + for be in BuildEnvironment.objects.all(): + be.needs_import = False + def _verify_be(): + is_changed = False + + def _update_sourcedir(): + be.sourcedir = os.environ.get('TOASTER_DIR') + return True + + if len(be.sourcedir) == 0: + is_changed = _update_sourcedir() + + if not be.sourcedir.startswith("/"): + print("\n -- Validation: The layers checkout directory must be set to an absolute path.") + is_changed = _update_sourcedir() + + if is_changed: + if be.betype == BuildEnvironment.TYPE_LOCAL: + be.needs_import = True + return True + + def _update_builddir(): + be.builddir = os.environ.get('TOASTER_DIR')+"/build" + return True + + if len(be.builddir) == 0: + is_changed = _update_builddir() + + if not be.builddir.startswith("/"): + print("\n -- Validation: The build directory must to be set to an absolute path.") + is_changed = _update_builddir() + + if is_changed: + print("\nBuild configuration saved") + be.save() + return True + + if be.needs_import: + try: + print("Loading default settings") + call_command("loaddata", "settings") + template_conf = os.environ.get("TEMPLATECONF", "") + + if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0: + # only use the custom settings + pass + elif "poky" in template_conf: + print("Loading poky configuration") + call_command("loaddata", "poky") + else: + print("Loading OE-Core configuration") + call_command("loaddata", "oe-core") + if template_conf: + oe_core_path = os.path.realpath( + template_conf + + "/../") + else: + print("TEMPLATECONF not found. You may have to" + " manually configure layer paths") + oe_core_path = input("Please enter the path of" + " your openembedded-core " + "layer: ") + # Update the layer instances of openemebedded-core + for layer in Layer.objects.filter( + name="openembedded-core", + local_source_dir="OE-CORE-LAYER-DIR"): + layer.local_path = oe_core_path + layer.save() + + # Import the custom fixture if it's present + with warnings.catch_warnings(): + warnings.filterwarnings( + action="ignore", + message="^.*No fixture named.*$") + print("Importing custom settings if present") + try: + call_command("loaddata", "custom") + except: + print("NOTE: optional fixture 'custom' not found") + + # we run lsupdates after config update + print("\nFetching information from the layer index, " + "please wait.\nYou can re-update any time later " + "by running bitbake/lib/toaster/manage.py " + "lsupdates\n") + call_command("lsupdates") + + # we don't look for any other config files + return is_changed + except Exception as e: + print("Failure while trying to setup toaster: %s" + % e) + traceback.print_exc() + + return is_changed + + while _verify_be(): + pass + return 0 + + def _verify_default_settings(self): + # verify that default settings are there + if ToasterSetting.objects.filter(name='DEFAULT_RELEASE').count() != 1: + ToasterSetting.objects.filter(name='DEFAULT_RELEASE').delete() + ToasterSetting.objects.get_or_create(name='DEFAULT_RELEASE', value='') + return 0 + + def _verify_builds_in_progress(self): + # we are just starting up. we must not have any builds in progress, or build environments taken + for b in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS): + BRError.objects.create(req=b, errtype="toaster", + errmsg= + "Toaster found this build IN PROGRESS while Toaster started up. This is an inconsistent state, and the build was marked as failed") + + BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS).update(state=BuildRequest.REQ_FAILED) + + BuildEnvironment.objects.update(lock=BuildEnvironment.LOCK_FREE) + + # also mark "In Progress builds as failures" + from django.utils import timezone + Build.objects.filter(outcome=Build.IN_PROGRESS).update(outcome=Build.FAILED, completed_on=timezone.now()) + + return 0 + + + + def handle(self, **options): + retval = 0 + retval += self._verify_build_environment() + retval += self._verify_default_settings() + retval += self._verify_builds_in_progress() + + return retval diff --git a/poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py new file mode 100644 index 000000000..791e53eab --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -0,0 +1,192 @@ +from django.core.management.base import BaseCommand +from django.db import transaction +from django.db.models import Q + +from bldcontrol.bbcontroller import getBuildEnvironmentController +from bldcontrol.models import BuildRequest, BuildEnvironment +from bldcontrol.models import BRError, BRVariable + +from orm.models import Build, LogMessage, Target + +import logging +import traceback +import signal +import os + +logger = logging.getLogger("toaster") + + +class Command(BaseCommand): + args = "" + help = "Schedules and executes build requests as possible. "\ + "Does not return (interrupt with Ctrl-C)" + + @transaction.atomic + def _selectBuildEnvironment(self): + bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE) + bec.be.lock = BuildEnvironment.LOCK_LOCK + bec.be.save() + return bec + + @transaction.atomic + def _selectBuildRequest(self): + br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first() + return br + + def schedule(self): + try: + # select the build environment and the request to build + br = self._selectBuildRequest() + if br: + br.state = BuildRequest.REQ_INPROGRESS + br.save() + else: + return + + try: + bec = self._selectBuildEnvironment() + except IndexError as e: + # we could not find a BEC; postpone the BR + br.state = BuildRequest.REQ_QUEUED + br.save() + logger.debug("runbuilds: No build env") + return + + logger.info("runbuilds: starting build %s, environment %s" % + (br, bec.be)) + + # let the build request know where it is being executed + br.environment = bec.be + br.save() + + # this triggers an async build + bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), + br.brvariable_set.all(), br.brtarget_set.all(), + "%d:%d" % (br.pk, bec.be.pk)) + + except Exception as e: + logger.error("runbuilds: Error launching build %s" % e) + traceback.print_exc() + if "[Errno 111] Connection refused" in str(e): + # Connection refused, read toaster_server.out + errmsg = bec.readServerLogFile() + else: + errmsg = str(e) + + BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg, + traceback=traceback.format_exc()) + br.state = BuildRequest.REQ_FAILED + br.save() + bec.be.lock = BuildEnvironment.LOCK_FREE + bec.be.save() + # Cancel the pending build and report the exception to the UI + log_object = LogMessage.objects.create( + build = br.build, + level = LogMessage.EXCEPTION, + message = errmsg) + log_object.save() + br.build.outcome = Build.FAILED + br.build.save() + + def archive(self): + for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE): + if br.build is None: + br.state = BuildRequest.REQ_FAILED + else: + br.state = BuildRequest.REQ_COMPLETED + br.save() + + def cleanup(self): + from django.utils import timezone + from datetime import timedelta + # environments locked for more than 30 seconds + # they should be unlocked + BuildEnvironment.objects.filter( + Q(buildrequest__state__in=[BuildRequest.REQ_FAILED, + BuildRequest.REQ_COMPLETED, + BuildRequest.REQ_CANCELLING]) & + Q(lock=BuildEnvironment.LOCK_LOCK) & + Q(updated__lt=timezone.now() - timedelta(seconds=30)) + ).update(lock=BuildEnvironment.LOCK_FREE) + + # update all Builds that were in progress and failed to start + for br in BuildRequest.objects.filter( + state=BuildRequest.REQ_FAILED, + build__outcome=Build.IN_PROGRESS): + # transpose the launch errors in ToasterExceptions + br.build.outcome = Build.FAILED + for brerror in br.brerror_set.all(): + logger.debug("Saving error %s" % brerror) + LogMessage.objects.create(build=br.build, + level=LogMessage.EXCEPTION, + message=brerror.errmsg) + br.build.save() + + # we don't have a true build object here; hence, toasterui + # didn't have a change to release the BE lock + br.environment.lock = BuildEnvironment.LOCK_FREE + br.environment.save() + + # update all BuildRequests without a build created + for br in BuildRequest.objects.filter(build=None): + br.build = Build.objects.create(project=br.project, + completed_on=br.updated, + started_on=br.created) + br.build.outcome = Build.FAILED + try: + br.build.machine = br.brvariable_set.get(name='MACHINE').value + except BRVariable.DoesNotExist: + pass + br.save() + # transpose target information + for brtarget in br.brtarget_set.all(): + Target.objects.create(build=br.build, + target=brtarget.target, + task=brtarget.task) + # transpose the launch errors in ToasterExceptions + for brerror in br.brerror_set.all(): + LogMessage.objects.create(build=br.build, + level=LogMessage.EXCEPTION, + message=brerror.errmsg) + + br.build.save() + + # Make sure the LOCK is removed for builds which have been fully + # cancelled + for br in BuildRequest.objects.filter( + Q(build__outcome=Build.CANCELLED) & + Q(state=BuildRequest.REQ_CANCELLING) & + ~Q(environment=None)): + br.environment.lock = BuildEnvironment.LOCK_FREE + br.environment.save() + + def runbuild(self): + try: + self.cleanup() + except Exception as e: + logger.warn("runbuilds: cleanup exception %s" % str(e)) + + try: + self.archive() + except Exception as e: + logger.warn("runbuilds: archive exception %s" % str(e)) + + try: + self.schedule() + except Exception as e: + logger.warn("runbuilds: schedule exception %s" % str(e)) + + def handle(self, **options): + pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."), + ".runbuilds.pid") + + with open(pidfile_path, 'w') as pidfile: + pidfile.write("%s" % os.getpid()) + + self.runbuild() + + signal.signal(signal.SIGUSR1, lambda sig, frame: None) + + while True: + signal.pause() + self.runbuild() diff --git a/poky/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py b/poky/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py new file mode 100644 index 000000000..67db37856 --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='BRBitbake', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('giturl', models.CharField(max_length=254)), + ('commit', models.CharField(max_length=254)), + ('dirpath', models.CharField(max_length=254)), + ], + ), + migrations.CreateModel( + name='BRError', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('errtype', models.CharField(max_length=100)), + ('errmsg', models.TextField()), + ('traceback', models.TextField()), + ], + ), + migrations.CreateModel( + name='BRLayer', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=100)), + ('giturl', models.CharField(max_length=254)), + ('commit', models.CharField(max_length=254)), + ('dirpath', models.CharField(max_length=254)), + ('layer_version', models.ForeignKey(to='orm.Layer_Version', null=True)), + ], + ), + migrations.CreateModel( + name='BRTarget', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('target', models.CharField(max_length=100)), + ('task', models.CharField(max_length=100, null=True)), + ], + ), + migrations.CreateModel( + name='BRVariable', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=100)), + ('value', models.TextField(blank=True)), + ], + ), + migrations.CreateModel( + name='BuildEnvironment', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('address', models.CharField(max_length=254)), + ('betype', models.IntegerField(choices=[(0, b'local'), (1, b'ssh')])), + ('bbaddress', models.CharField(max_length=254, blank=True)), + ('bbport', models.IntegerField(default=-1)), + ('bbtoken', models.CharField(max_length=126, blank=True)), + ('bbstate', models.IntegerField(default=0, choices=[(0, b'stopped'), (1, b'started')])), + ('sourcedir', models.CharField(max_length=512, blank=True)), + ('builddir', models.CharField(max_length=512, blank=True)), + ('lock', models.IntegerField(default=0, choices=[(0, b'free'), (1, b'lock'), (2, b'running')])), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='BuildRequest', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('state', models.IntegerField(default=0, choices=[(0, b'created'), (1, b'queued'), (2, b'in progress'), (3, b'completed'), (4, b'failed'), (5, b'deleted'), (6, b'archive')])), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('build', models.OneToOneField(null=True, to='orm.Build')), + ('environment', models.ForeignKey(to='bldcontrol.BuildEnvironment', null=True)), + ('project', models.ForeignKey(to='orm.Project')), + ], + ), + migrations.AddField( + model_name='brvariable', + name='req', + field=models.ForeignKey(to='bldcontrol.BuildRequest'), + ), + migrations.AddField( + model_name='brtarget', + name='req', + field=models.ForeignKey(to='bldcontrol.BuildRequest'), + ), + migrations.AddField( + model_name='brlayer', + name='req', + field=models.ForeignKey(to='bldcontrol.BuildRequest'), + ), + migrations.AddField( + model_name='brerror', + name='req', + field=models.ForeignKey(to='bldcontrol.BuildRequest'), + ), + migrations.AddField( + model_name='brbitbake', + name='req', + field=models.OneToOneField(to='bldcontrol.BuildRequest'), + ), + ] diff --git a/poky/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py b/poky/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py new file mode 100644 index 000000000..0c2475aba --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='buildenvironment', + name='betype', + field=models.IntegerField(choices=[(0, b'local')]), + ), + ] diff --git a/poky/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py b/poky/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py new file mode 100644 index 000000000..eec9216ca --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0002_auto_20160120_1250'), + ] + + operations = [ + migrations.AlterField( + model_name='buildrequest', + name='state', + field=models.IntegerField(default=0, choices=[(0, b'created'), (1, b'queued'), (2, b'in progress'), (3, b'completed'), (4, b'failed'), (5, b'deleted'), (6, b'cancelling'), (7, b'archive')]), + ), + ] diff --git a/poky/bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py b/poky/bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py new file mode 100644 index 000000000..3d9062954 --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0003_add_cancelling_state'), + ] + + operations = [ + migrations.AlterField( + model_name='buildenvironment', + name='bbstate', + field=models.IntegerField(default=0, choices=[(0, 'stopped'), (1, 'started')]), + ), + migrations.AlterField( + model_name='buildenvironment', + name='betype', + field=models.IntegerField(choices=[(0, 'local')]), + ), + migrations.AlterField( + model_name='buildenvironment', + name='lock', + field=models.IntegerField(default=0, choices=[(0, 'free'), (1, 'lock'), (2, 'running')]), + ), + migrations.AlterField( + model_name='buildrequest', + name='state', + field=models.IntegerField(default=0, choices=[(0, 'created'), (1, 'queued'), (2, 'in progress'), (3, 'completed'), (4, 'failed'), (5, 'deleted'), (6, 'cancelling'), (7, 'archive')]), + ), + ] diff --git a/poky/bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py b/poky/bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py new file mode 100644 index 000000000..4bb951776 --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0004_auto_20160523_1446'), + ] + + operations = [ + migrations.AlterField( + model_name='buildrequest', + name='state', + field=models.IntegerField(choices=[(0, 'created'), (1, 'queued'), (2, 'in progress'), (3, 'failed'), (4, 'deleted'), (5, 'cancelling'), (6, 'completed'), (7, 'archive')], default=0), + ), + ] diff --git a/poky/bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py b/poky/bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py new file mode 100644 index 000000000..2460002f0 --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0005_reorder_buildrequest_states'), + ] + + operations = [ + migrations.AddField( + model_name='brlayer', + name='local_source_dir', + field=models.CharField(max_length=254, null=True), + ), + ] diff --git a/poky/bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py b/poky/bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py new file mode 100644 index 000000000..4be42a4cf --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0006_brlayer_local_source_dir'), + ] + + operations = [ + migrations.AlterField( + model_name='brlayer', + name='commit', + field=models.CharField(max_length=254, null=True), + ), + migrations.AlterField( + model_name='brlayer', + name='dirpath', + field=models.CharField(max_length=254, null=True), + ), + migrations.AlterField( + model_name='brlayer', + name='giturl', + field=models.CharField(max_length=254, null=True), + ), + ] diff --git a/poky/bitbake/lib/toaster/bldcontrol/migrations/__init__.py b/poky/bitbake/lib/toaster/bldcontrol/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/bldcontrol/models.py b/poky/bitbake/lib/toaster/bldcontrol/models.py new file mode 100644 index 000000000..409614b9e --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/models.py @@ -0,0 +1,162 @@ +from __future__ import unicode_literals +from django.db import models +from django.core.validators import MaxValueValidator, MinValueValidator +from django.utils.encoding import force_text +from orm.models import Project, ProjectLayer, ProjectVariable, ProjectTarget, Build, Layer_Version + +import logging +logger = logging.getLogger("toaster") +# a BuildEnvironment is the equivalent of the "build/" directory on the localhost +class BuildEnvironment(models.Model): + SERVER_STOPPED = 0 + SERVER_STARTED = 1 + SERVER_STATE = ( + (SERVER_STOPPED, "stopped"), + (SERVER_STARTED, "started"), + ) + + TYPE_LOCAL = 0 + TYPE = ( + (TYPE_LOCAL, "local"), + ) + + LOCK_FREE = 0 + LOCK_LOCK = 1 + LOCK_RUNNING = 2 + LOCK_STATE = ( + (LOCK_FREE, "free"), + (LOCK_LOCK, "lock"), + (LOCK_RUNNING, "running"), + ) + + address = models.CharField(max_length = 254) + betype = models.IntegerField(choices = TYPE) + bbaddress = models.CharField(max_length = 254, blank = True) + bbport = models.IntegerField(default = -1) + bbtoken = models.CharField(max_length = 126, blank = True) + bbstate = models.IntegerField(choices = SERVER_STATE, default = SERVER_STOPPED) + sourcedir = models.CharField(max_length = 512, blank = True) + builddir = models.CharField(max_length = 512, blank = True) + lock = models.IntegerField(choices = LOCK_STATE, default = LOCK_FREE) + created = models.DateTimeField(auto_now_add = True) + updated = models.DateTimeField(auto_now = True) + + def get_artifact(self, path): + if self.betype == BuildEnvironment.TYPE_LOCAL: + return open(path, "r") + raise NotImplementedError("FIXME: artifact download not implemented "\ + "for build environment type %s" % \ + self.get_betype_display()) + + def has_artifact(self, path): + import os + if self.betype == BuildEnvironment.TYPE_LOCAL: + return os.path.exists(path) + raise NotImplementedError("FIXME: has artifact not implemented for "\ + "build environment type %s" % \ + self.get_betype_display()) + +# a BuildRequest is a request that the scheduler will build using a BuildEnvironment +# the build request queue is the table itself, ordered by state + +class BuildRequest(models.Model): + REQ_CREATED = 0 + REQ_QUEUED = 1 + REQ_INPROGRESS = 2 + REQ_FAILED = 3 + REQ_DELETED = 4 + REQ_CANCELLING = 5 + REQ_COMPLETED = 6 + REQ_ARCHIVE = 7 + + REQUEST_STATE = ( + (REQ_CREATED, "created"), + (REQ_QUEUED, "queued"), + (REQ_INPROGRESS, "in progress"), + (REQ_FAILED, "failed"), + (REQ_DELETED, "deleted"), + (REQ_CANCELLING, "cancelling"), + (REQ_COMPLETED, "completed"), + (REQ_ARCHIVE, "archive"), + ) + + search_allowed_fields = ("brtarget__target", "build__project__name") + + project = models.ForeignKey(Project) + build = models.OneToOneField(Build, null = True) # TODO: toasterui should set this when Build is created + environment = models.ForeignKey(BuildEnvironment, null = True) + state = models.IntegerField(choices = REQUEST_STATE, default = REQ_CREATED) + created = models.DateTimeField(auto_now_add = True) + updated = models.DateTimeField(auto_now = True) + + def __init__(self, *args, **kwargs): + super(BuildRequest, self).__init__(*args, **kwargs) + # Save the old state in case it's about to be modified + self.old_state = self.state + + def save(self, *args, **kwargs): + # Check that the state we're trying to set is not going backwards + # e.g. from REQ_FAILED to REQ_INPROGRESS + if self.old_state != self.state and self.old_state > self.state: + logger.warning("Invalid state change requested: " + "Cannot go from %s to %s - ignoring request" % + (BuildRequest.REQUEST_STATE[self.old_state][1], + BuildRequest.REQUEST_STATE[self.state][1]) + ) + # Set property back to the old value + self.state = self.old_state + return + + super(BuildRequest, self).save(*args, **kwargs) + + + def get_duration(self): + return (self.updated - self.created).total_seconds() + + def get_sorted_target_list(self): + tgts = self.brtarget_set.order_by( 'target' ); + return( tgts ); + + def get_machine(self): + return self.brvariable_set.get(name="MACHINE").value + + def __str__(self): + return force_text('%s %s' % (self.project, self.get_state_display())) + +# These tables specify the settings for running an actual build. +# They MUST be kept in sync with the tables in orm.models.Project* + + +class BRLayer(models.Model): + req = models.ForeignKey(BuildRequest) + name = models.CharField(max_length=100) + giturl = models.CharField(max_length=254, null=True) + local_source_dir = models.CharField(max_length=254, null=True) + commit = models.CharField(max_length=254, null=True) + dirpath = models.CharField(max_length=254, null=True) + layer_version = models.ForeignKey(Layer_Version, null=True) + +class BRBitbake(models.Model): + req = models.OneToOneField(BuildRequest) # only one bitbake for a request + giturl = models.CharField(max_length =254) + commit = models.CharField(max_length = 254) + dirpath = models.CharField(max_length = 254) + +class BRVariable(models.Model): + req = models.ForeignKey(BuildRequest) + name = models.CharField(max_length=100) + value = models.TextField(blank = True) + +class BRTarget(models.Model): + req = models.ForeignKey(BuildRequest) + target = models.CharField(max_length=100) + task = models.CharField(max_length=100, null=True) + +class BRError(models.Model): + req = models.ForeignKey(BuildRequest) + errtype = models.CharField(max_length=100) + errmsg = models.TextField() + traceback = models.TextField() + + def __str__(self): + return "%s (%s)" % (self.errmsg, self.req) diff --git a/poky/bitbake/lib/toaster/bldcontrol/views.py b/poky/bitbake/lib/toaster/bldcontrol/views.py new file mode 100644 index 000000000..60f00ef0e --- /dev/null +++ b/poky/bitbake/lib/toaster/bldcontrol/views.py @@ -0,0 +1 @@ +# Create your views here. diff --git a/poky/bitbake/lib/toaster/manage.py b/poky/bitbake/lib/toaster/manage.py new file mode 100755 index 000000000..0c7ea5088 --- /dev/null +++ b/poky/bitbake/lib/toaster/manage.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toastermain.settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/poky/bitbake/lib/toaster/orm/__init__.py b/poky/bitbake/lib/toaster/orm/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/orm/fixtures/README b/poky/bitbake/lib/toaster/orm/fixtures/README new file mode 100644 index 000000000..1b1c660aa --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/fixtures/README @@ -0,0 +1,30 @@ +# Fixtures directory + +Fixtures are data dumps that can be loaded into Toaster's database to provide +configuration and data. + +In this directory we have the fixtures which are loaded the first time you start Toaster. +This is to provide useful default values and metadata to Toaster. + + - settings.xml This Contains Toaster wide settings, such as the default values for + certain bitbake variables. + + - poky.xml This is the default release data for supported poky based setup + + - oe-core.xml This is the default release data for supported oe-core based setups + +# Custom data/configuration + + - custom.xml + +To add custom initial data/configuration to Toaster place a file called +"custom.xml" in this directory. If present it will be loaded into the database. +We suggest that this is used to overlay any configuration already done. +All objects loaded with the same primary keys overwrite the existing data. +Data can be provided in XML, JSON and if installed YAML formats. + +# To load data at any point in time + +Use the django management command manage.py loaddata +For further information see the Django command documentation at: +https://docs.djangoproject.com/en/1.8/ref/django-admin/#django-admin-loaddata diff --git a/poky/bitbake/lib/toaster/orm/fixtures/custom_toaster_append.sh_sample b/poky/bitbake/lib/toaster/orm/fixtures/custom_toaster_append.sh_sample new file mode 100755 index 000000000..8c4e16316 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/fixtures/custom_toaster_append.sh_sample @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright (C) 2017 Intel Corp. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# This is sample software. Rename it to 'custom_toaster_append.sh' and +# enable the respective custom sections. + +verbose=0 +if [ $verbose -ne 0 ] ; then + echo "custom_toaster_append.sh:$*" +fi + +if [ "toaster_prepend" = "$1" ] ; then + echo "Add custom actions here when Toaster script is started" +fi + +if [ "web_start_postpend" = "$1" ] ; then + echo "Add custom actions here after Toaster web service is started" +fi + +if [ "web_stop_postpend" = "$1" ] ; then + echo "Add custom actions here after Toaster web service is stopped" +fi + +if [ "noweb_start_postpend" = "$1" ] ; then + echo "Add custom actions here after Toaster (no web) service is started" +fi + +if [ "noweb_stop_postpend" = "$1" ] ; then + echo "Add custom actions here after Toaster (no web) service is stopped" +fi + +if [ "toaster_postpend" = "$1" ] ; then + echo "Add custom actions here after Toaster script is done" +fi + diff --git a/poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml b/poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml new file mode 100644 index 000000000..d7ea78dc2 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml @@ -0,0 +1,97 @@ + + + + + DEFCONF_DISTRO + nodistro + + + + + sumo + git://git.openembedded.org/bitbake + 1.38 + + + HEAD + git://git.openembedded.org/bitbake + HEAD + + + master + git://git.openembedded.org/bitbake + master + + + rocko + git://git.openembedded.org/bitbake + 1.36 + + + + + rocko + Openembedded Sumo + 1 + sumo + Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=sumo\">OpenEmbedded Sumo</a> branch. + + + local + Local Openembedded + 2 + HEAD + Toaster will run your builds with the version of OpenEmbedded that you have cloned or downloaded to your computer. + + + master + OpenEmbedded core master + 3 + master + Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/\">OpenEmbedded master</a> branch. + + + rocko + Openembedded Rocko + 1 + rocko + Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=rocko\">OpenEmbedded Rocko</a> branch. + + + + + 1 + openembedded-core + + + 2 + openembedded-core + + + 3 + openembedded-core + + + 4 + openembedded-core + + + + + + openembedded-core + git://git.openembedded.org/openembedded-core + http://cgit.openembedded.org/openembedded-core + http://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch% + http://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch% + + + 1 + 2 + OE-CORE-LAYER-DIR + HEAD + meta + 0 + + + diff --git a/poky/bitbake/lib/toaster/orm/fixtures/poky.xml b/poky/bitbake/lib/toaster/orm/fixtures/poky.xml new file mode 100644 index 000000000..6c966da4a --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/fixtures/poky.xml @@ -0,0 +1,234 @@ + + + + + DEFCONF_DISTRO + poky + + + + + sumo + git://git.yoctoproject.org/poky + sumo + bitbake + + + HEAD + git://git.yoctoproject.org/poky + HEAD + bitbake + + + master + git://git.yoctoproject.org/poky + master + bitbake + + + rocko + git://git.yoctoproject.org/poky + rocko + bitbake + + + + + + sumo + Yocto Project 2.5 "Sumo" + 1 + sumo + Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=sumo">Yocto Project Sumo branch</a>. + + + local + Local Yocto Project + 2 + HEAD + Toaster will run your builds with the version of the Yocto Project you have cloned or downloaded to your computer. + + + master + Yocto Project master + 3 + master + Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/">Yocto Project Master branch</a>. + + + rocko + Yocto Project 2.4 "Rocko" + 1 + rocko + Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=rocko">Yocto Project Rocko branch</a>. + + + + + 1 + openembedded-core + + + 1 + meta-poky + + + 1 + meta-yocto-bsp + + + 2 + openembedded-core + + + 2 + meta-poky + + + 2 + meta-yocto-bsp + + + 3 + openembedded-core + + + 3 + meta-poky + + + 3 + meta-yocto-bsp + + + 4 + openembedded-core + + + 4 + meta-poky + + + 4 + meta-yocto-bsp + + + + + openembedded-core + + git://git.yoctoproject.org/poky + http://git.yoctoproject.org/cgit/cgit.cgi/poky + http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch% + http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch% + + + 1 + 0 + 1 + sumo + meta + + + 1 + 0 + 2 + HEAD + HEAD + meta + + + 1 + 0 + 3 + master + meta + + + 1 + 0 + 4 + rocko + meta + + + + meta-poky + + git://git.yoctoproject.org/poky + http://git.yoctoproject.org/cgit/cgit.cgi/poky + http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch% + http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch% + + + 2 + 0 + 1 + sumo + meta-poky + + + 2 + 0 + 2 + HEAD + HEAD + meta-poky + + + 2 + 0 + 3 + master + meta-poky + + + 2 + 0 + 4 + rocko + meta-poky + + + + meta-yocto-bsp + + git://git.yoctoproject.org/poky + http://git.yoctoproject.org/cgit/cgit.cgi/poky + http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch% + http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch% + + + 3 + 0 + 1 + sumo + meta-yocto-bsp + + + 3 + 0 + 2 + HEAD + HEAD + meta-yocto-bsp + + + 3 + 0 + 3 + master + meta-yocto-bsp + + + 3 + 0 + 4 + rocko + meta-yocto-bsp + + diff --git a/poky/bitbake/lib/toaster/orm/fixtures/settings.xml b/poky/bitbake/lib/toaster/orm/fixtures/settings.xml new file mode 100644 index 000000000..78c0fdca7 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/fixtures/settings.xml @@ -0,0 +1,33 @@ + + + + + + DEFAULT_RELEASE + master + + + DEFCONF_PACKAGE_CLASSES + package_rpm + + + DEFCONF_MACHINE + qemux86 + + + DEFCONF_SSTATE_DIR + ${TOPDIR}/../sstate-cache + + + DEFCONF_IMAGE_INSTALL_append + + + + DEFCONF_IMAGE_FSTYPES + ext3 jffs2 tar.bz2 + + + DEFCONF_DL_DIR + ${TOPDIR}/../downloads + + diff --git a/poky/bitbake/lib/toaster/orm/management/__init__.py b/poky/bitbake/lib/toaster/orm/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/orm/management/commands/__init__.py b/poky/bitbake/lib/toaster/orm/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py new file mode 100644 index 000000000..efc6b3a94 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py @@ -0,0 +1,337 @@ +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2016-2017 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from django.core.management.base import BaseCommand + +from orm.models import LayerSource, Layer, Release, Layer_Version +from orm.models import LayerVersionDependency, Machine, Recipe +from orm.models import Distro +from orm.models import ToasterSetting + +import os +import sys + +import json +import logging +import threading +import time +logger = logging.getLogger("toaster") + +DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/" + + +class Spinner(threading.Thread): + """ A simple progress spinner to indicate download/parsing is happening""" + def __init__(self, *args, **kwargs): + super(Spinner, self).__init__(*args, **kwargs) + self.setDaemon(True) + self.signal = True + + def run(self): + os.system('setterm -cursor off') + while self.signal: + for char in ["/", "-", "\\", "|"]: + sys.stdout.write("\r" + char) + sys.stdout.flush() + time.sleep(0.25) + os.system('setterm -cursor on') + + def stop(self): + self.signal = False + + +class Command(BaseCommand): + args = "" + help = "Updates locally cached information from a layerindex server" + + def mini_progress(self, what, i, total): + i = i + 1 + pec = (float(i)/float(total))*100 + + sys.stdout.write("\rUpdating %s %d%%" % + (what, + pec)) + sys.stdout.flush() + if int(pec) is 100: + sys.stdout.write("\n") + sys.stdout.flush() + + def update(self): + """ + Fetches layer, recipe and machine information from a layerindex + server + """ + os.system('setterm -cursor off') + + self.apiurl = DEFAULT_LAYERINDEX_SERVER + if ToasterSetting.objects.filter(name='CUSTOM_LAYERINDEX_SERVER').count() == 1: + self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value + + assert self.apiurl is not None + try: + from urllib.request import urlopen, URLError + from urllib.parse import urlparse + except ImportError: + from urllib2 import urlopen, URLError + from urlparse import urlparse + + proxy_settings = os.environ.get("http_proxy", None) + + def _get_json_response(apiurl=None): + if None == apiurl: + apiurl=self.apiurl + http_progress = Spinner() + http_progress.start() + + _parsedurl = urlparse(apiurl) + path = _parsedurl.path + + # logger.debug("Fetching %s", apiurl) + try: + res = urlopen(apiurl) + except URLError as e: + raise Exception("Failed to read %s: %s" % (path, e.reason)) + + parsed = json.loads(res.read().decode('utf-8')) + + http_progress.stop() + return parsed + + # verify we can get the basic api + try: + apilinks = _get_json_response() + except Exception as e: + import traceback + if proxy_settings is not None: + logger.info("EE: Using proxy %s" % proxy_settings) + logger.warning("EE: could not connect to %s, skipping update:" + "%s\n%s" % (self.apiurl, e, traceback.format_exc())) + return + + # update branches; only those that we already have names listed in the + # Releases table + whitelist_branch_names = [rel.branch_name + for rel in Release.objects.all()] + if len(whitelist_branch_names) == 0: + raise Exception("Failed to make list of branches to fetch") + + logger.info("Fetching metadata releases for %s", + " ".join(whitelist_branch_names)) + + branches_info = _get_json_response(apilinks['branches'] + + "?filter=name:%s" + % "OR".join(whitelist_branch_names)) + + # Map the layer index branches to toaster releases + li_branch_id_to_toaster_release = {} + + total = len(branches_info) + for i, branch in enumerate(branches_info): + li_branch_id_to_toaster_release[branch['id']] = \ + Release.objects.get(name=branch['name']) + self.mini_progress("Releases", i, total) + + # keep a track of the layerindex (li) id mappings so that + # layer_versions can be created for these layers later on + li_layer_id_to_toaster_layer_id = {} + + logger.info("Fetching layers") + + layers_info = _get_json_response(apilinks['layerItems']) + + total = len(layers_info) + for i, li in enumerate(layers_info): + try: + l, created = Layer.objects.get_or_create(name=li['name']) + l.up_date = li['updated'] + l.summary = li['summary'] + l.description = li['description'] + + if created: + # predefined layers in the fixtures (for example poky.xml) + # always preempt the Layer Index for these values + l.vcs_url = li['vcs_url'] + l.vcs_web_url = li['vcs_web_url'] + l.vcs_web_tree_base_url = li['vcs_web_tree_base_url'] + l.vcs_web_file_base_url = li['vcs_web_file_base_url'] + l.save() + except Layer.MultipleObjectsReturned: + logger.info("Skipped %s as we found multiple layers and " + "don't know which to update" % + li['name']) + + li_layer_id_to_toaster_layer_id[li['id']] = l.pk + + self.mini_progress("layers", i, total) + + # update layer_versions + logger.info("Fetching layer versions") + layerbranches_info = _get_json_response( + apilinks['layerBranches'] + "?filter=branch__name:%s" % + "OR".join(whitelist_branch_names)) + + # Map Layer index layer_branch object id to + # layer_version toaster object id + li_layer_branch_id_to_toaster_lv_id = {} + + total = len(layerbranches_info) + for i, lbi in enumerate(layerbranches_info): + # release as defined by toaster map to layerindex branch + release = li_branch_id_to_toaster_release[lbi['branch']] + + try: + lv, created = Layer_Version.objects.get_or_create( + layer=Layer.objects.get( + pk=li_layer_id_to_toaster_layer_id[lbi['layer']]), + release=release + ) + except KeyError: + logger.warning( + "No such layerindex layer referenced by layerbranch %d" % + lbi['layer']) + continue + + if created: + lv.release = li_branch_id_to_toaster_release[lbi['branch']] + lv.up_date = lbi['updated'] + lv.commit = lbi['actual_branch'] + lv.dirpath = lbi['vcs_subdir'] + lv.save() + + li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\ + lv.pk + self.mini_progress("layer versions", i, total) + + logger.info("Fetching layer version dependencies") + # update layer dependencies + layerdependencies_info = _get_json_response( + apilinks['layerDependencies'] + + "?filter=layerbranch__branch__name:%s" % + "OR".join(whitelist_branch_names)) + + dependlist = {} + for ldi in layerdependencies_info: + try: + lv = Layer_Version.objects.get( + pk=li_layer_branch_id_to_toaster_lv_id[ldi['layerbranch']]) + except Layer_Version.DoesNotExist as e: + continue + + if lv not in dependlist: + dependlist[lv] = [] + try: + layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']] + + dependlist[lv].append( + Layer_Version.objects.get(layer__pk=layer_id, + release=lv.release)) + + except Layer_Version.DoesNotExist: + logger.warning("Cannot find layer version (ls:%s)," + "up_id:%s lv:%s" % + (self, ldi['dependency'], lv)) + + total = len(dependlist) + for i, lv in enumerate(dependlist): + LayerVersionDependency.objects.filter(layer_version=lv).delete() + for lvd in dependlist[lv]: + LayerVersionDependency.objects.get_or_create(layer_version=lv, + depends_on=lvd) + self.mini_progress("Layer version dependencies", i, total) + + # update Distros + logger.info("Fetching distro information") + distros_info = _get_json_response( + apilinks['distros'] + "?filter=layerbranch__branch__name:%s" % + "OR".join(whitelist_branch_names)) + + total = len(distros_info) + for i, di in enumerate(distros_info): + distro, created = Distro.objects.get_or_create( + name=di['name'], + layer_version=Layer_Version.objects.get( + pk=li_layer_branch_id_to_toaster_lv_id[di['layerbranch']])) + distro.up_date = di['updated'] + distro.name = di['name'] + distro.description = di['description'] + distro.save() + self.mini_progress("distros", i, total) + + # update machines + logger.info("Fetching machine information") + machines_info = _get_json_response( + apilinks['machines'] + "?filter=layerbranch__branch__name:%s" % + "OR".join(whitelist_branch_names)) + + total = len(machines_info) + for i, mi in enumerate(machines_info): + mo, created = Machine.objects.get_or_create( + name=mi['name'], + layer_version=Layer_Version.objects.get( + pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']])) + mo.up_date = mi['updated'] + mo.name = mi['name'] + mo.description = mi['description'] + mo.save() + self.mini_progress("machines", i, total) + + # update recipes; paginate by layer version / layer branch + logger.info("Fetching recipe information") + recipes_info = _get_json_response( + apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" % + "OR".join(whitelist_branch_names)) + + total = len(recipes_info) + for i, ri in enumerate(recipes_info): + try: + lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']] + lv = Layer_Version.objects.get(pk=lv_id) + + ro, created = Recipe.objects.get_or_create( + layer_version=lv, + name=ri['pn'] + ) + + ro.layer_version = lv + ro.up_date = ri['updated'] + ro.name = ri['pn'] + ro.version = ri['pv'] + ro.summary = ri['summary'] + ro.description = ri['description'] + ro.section = ri['section'] + ro.license = ri['license'] + ro.homepage = ri['homepage'] + ro.bugtracker = ri['bugtracker'] + ro.file_path = ri['filepath'] + "/" + ri['filename'] + if 'inherits' in ri: + ro.is_image = 'image' in ri['inherits'].split() + else: # workaround for old style layer index + ro.is_image = "-image-" in ri['pn'] + ro.save() + except Exception as e: + logger.warning("Failed saving recipe %s", e) + + self.mini_progress("recipes", i, total) + + os.system('setterm -cursor on') + + def handle(self, **options): + self.update() diff --git a/poky/bitbake/lib/toaster/orm/migrations/0001_initial.py b/poky/bitbake/lib/toaster/orm/migrations/0001_initial.py new file mode 100644 index 000000000..760462f6b --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0001_initial.py @@ -0,0 +1,504 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='BitbakeVersion', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(unique=True, max_length=32)), + ('giturl', models.URLField()), + ('branch', models.CharField(max_length=32)), + ('dirpath', models.CharField(max_length=255)), + ], + ), + migrations.CreateModel( + name='Branch', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('up_id', models.IntegerField(default=None, null=True)), + ('up_date', models.DateTimeField(default=None, null=True)), + ('name', models.CharField(max_length=50)), + ('short_description', models.CharField(max_length=50, blank=True)), + ], + options={ + 'verbose_name_plural': 'Branches', + }, + ), + migrations.CreateModel( + name='Build', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('machine', models.CharField(max_length=100)), + ('distro', models.CharField(max_length=100)), + ('distro_version', models.CharField(max_length=100)), + ('started_on', models.DateTimeField()), + ('completed_on', models.DateTimeField()), + ('outcome', models.IntegerField(default=2, choices=[(0, b'Succeeded'), (1, b'Failed'), (2, b'In Progress')])), + ('cooker_log_path', models.CharField(max_length=500)), + ('build_name', models.CharField(max_length=100)), + ('bitbake_version', models.CharField(max_length=50)), + ], + ), + migrations.CreateModel( + name='BuildArtifact', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('file_name', models.FilePathField()), + ('file_size', models.IntegerField()), + ('build', models.ForeignKey(to='orm.Build')), + ], + ), + migrations.CreateModel( + name='HelpText', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('area', models.IntegerField(choices=[(0, b'variable')])), + ('key', models.CharField(max_length=100)), + ('text', models.TextField()), + ('build', models.ForeignKey(related_name='helptext_build', to='orm.Build')), + ], + ), + migrations.CreateModel( + name='Layer', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('up_id', models.IntegerField(default=None, null=True)), + ('up_date', models.DateTimeField(default=None, null=True)), + ('name', models.CharField(max_length=100)), + ('layer_index_url', models.URLField()), + ('vcs_url', models.URLField(default=None, null=True)), + ('vcs_web_url', models.URLField(default=None, null=True)), + ('vcs_web_tree_base_url', models.URLField(default=None, null=True)), + ('vcs_web_file_base_url', models.URLField(default=None, null=True)), + ('summary', models.TextField(default=None, help_text=b'One-line description of the layer', null=True)), + ('description', models.TextField(default=None, null=True)), + ], + ), + migrations.CreateModel( + name='Layer_Version', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('up_id', models.IntegerField(default=None, null=True)), + ('up_date', models.DateTimeField(default=None, null=True)), + ('branch', models.CharField(max_length=80)), + ('commit', models.CharField(max_length=100)), + ('dirpath', models.CharField(default=None, max_length=255, null=True)), + ('priority', models.IntegerField(default=0)), + ('local_path', models.FilePathField(default=b'/', max_length=1024)), + ('build', models.ForeignKey(related_name='layer_version_build', default=None, to='orm.Build', null=True)), + ('layer', models.ForeignKey(related_name='layer_version_layer', to='orm.Layer')), + ], + ), + migrations.CreateModel( + name='LayerSource', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(unique=True, max_length=63)), + ('sourcetype', models.IntegerField(choices=[(0, b'local'), (1, b'layerindex'), (2, b'imported')])), + ('apiurl', models.CharField(default=None, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='LayerVersionDependency', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('up_id', models.IntegerField(default=None, null=True)), + ('depends_on', models.ForeignKey(related_name='dependees', to='orm.Layer_Version')), + ('layer_source', models.ForeignKey(default=None, to='orm.LayerSource', null=True)), + ('layer_version', models.ForeignKey(related_name='dependencies', to='orm.Layer_Version')), + ], + ), + migrations.CreateModel( + name='LogMessage', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('level', models.IntegerField(default=0, choices=[(0, b'info'), (1, b'warn'), (2, b'error'), (3, b'critical'), (-1, b'toaster exception')])), + ('message', models.TextField(null=True, blank=True)), + ('pathname', models.FilePathField(max_length=255, blank=True)), + ('lineno', models.IntegerField(null=True)), + ('build', models.ForeignKey(to='orm.Build')), + ], + ), + migrations.CreateModel( + name='Machine', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('up_id', models.IntegerField(default=None, null=True)), + ('up_date', models.DateTimeField(default=None, null=True)), + ('name', models.CharField(max_length=255)), + ('description', models.CharField(max_length=255)), + ('layer_source', models.ForeignKey(default=None, to='orm.LayerSource', null=True)), + ('layer_version', models.ForeignKey(to='orm.Layer_Version')), + ], + ), + migrations.CreateModel( + name='Package', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=100)), + ('installed_name', models.CharField(default=b'', max_length=100)), + ('version', models.CharField(max_length=100, blank=True)), + ('revision', models.CharField(max_length=32, blank=True)), + ('summary', models.TextField(blank=True)), + ('description', models.TextField(blank=True)), + ('size', models.IntegerField(default=0)), + ('installed_size', models.IntegerField(default=0)), + ('section', models.CharField(max_length=80, blank=True)), + ('license', models.CharField(max_length=80, blank=True)), + ('build', models.ForeignKey(to='orm.Build', null=True)), + ], + ), + migrations.CreateModel( + name='Package_Dependency', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('dep_type', models.IntegerField(choices=[(0, b'depends'), (1, b'depends'), (3, b'recommends'), (2, b'recommends'), (4, b'suggests'), (5, b'provides'), (6, b'replaces'), (7, b'conflicts')])), + ('depends_on', models.ForeignKey(related_name='package_dependencies_target', to='orm.Package')), + ('package', models.ForeignKey(related_name='package_dependencies_source', to='orm.Package')), + ], + ), + migrations.CreateModel( + name='Package_File', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('path', models.FilePathField(max_length=255, blank=True)), + ('size', models.IntegerField()), + ('package', models.ForeignKey(related_name='buildfilelist_package', to='orm.Package')), + ], + ), + migrations.CreateModel( + name='Project', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=100)), + ('short_description', models.CharField(max_length=50, blank=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('user_id', models.IntegerField(null=True)), + ('is_default', models.BooleanField(default=False)), + ('bitbake_version', models.ForeignKey(to='orm.BitbakeVersion', null=True)), + ], + ), + migrations.CreateModel( + name='ProjectLayer', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('optional', models.BooleanField(default=True)), + ('layercommit', models.ForeignKey(to='orm.Layer_Version', null=True)), + ('project', models.ForeignKey(to='orm.Project')), + ], + ), + migrations.CreateModel( + name='ProjectTarget', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('target', models.CharField(max_length=100)), + ('task', models.CharField(max_length=100, null=True)), + ('project', models.ForeignKey(to='orm.Project')), + ], + ), + migrations.CreateModel( + name='ProjectVariable', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=100)), + ('value', models.TextField(blank=True)), + ('project', models.ForeignKey(to='orm.Project')), + ], + ), + migrations.CreateModel( + name='Recipe', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('up_id', models.IntegerField(default=None, null=True)), + ('up_date', models.DateTimeField(default=None, null=True)), + ('name', models.CharField(max_length=100, blank=True)), + ('version', models.CharField(max_length=100, blank=True)), + ('summary', models.TextField(blank=True)), + ('description', models.TextField(blank=True)), + ('section', models.CharField(max_length=100, blank=True)), + ('license', models.CharField(max_length=200, blank=True)), + ('homepage', models.URLField(blank=True)), + ('bugtracker', models.URLField(blank=True)), + ('file_path', models.FilePathField(max_length=255)), + ('pathflags', models.CharField(max_length=200, blank=True)), + ('is_image', models.BooleanField(default=False)), + ('layer_source', models.ForeignKey(default=None, to='orm.LayerSource', null=True)), + ('layer_version', models.ForeignKey(related_name='recipe_layer_version', to='orm.Layer_Version')), + ], + ), + migrations.CreateModel( + name='Recipe_Dependency', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('dep_type', models.IntegerField(choices=[(0, b'depends'), (1, b'rdepends')])), + ('depends_on', models.ForeignKey(related_name='r_dependencies_depends', to='orm.Recipe')), + ('recipe', models.ForeignKey(related_name='r_dependencies_recipe', to='orm.Recipe')), + ], + ), + migrations.CreateModel( + name='Release', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(unique=True, max_length=32)), + ('description', models.CharField(max_length=255)), + ('branch_name', models.CharField(default=b'', max_length=50)), + ('helptext', models.TextField(null=True)), + ('bitbake_version', models.ForeignKey(to='orm.BitbakeVersion')), + ], + ), + migrations.CreateModel( + name='ReleaseDefaultLayer', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('layer_name', models.CharField(default=b'', max_length=100)), + ('release', models.ForeignKey(to='orm.Release')), + ], + ), + migrations.CreateModel( + name='ReleaseLayerSourcePriority', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('priority', models.IntegerField(default=0)), + ('layer_source', models.ForeignKey(to='orm.LayerSource')), + ('release', models.ForeignKey(to='orm.Release')), + ], + ), + migrations.CreateModel( + name='Target', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('target', models.CharField(max_length=100)), + ('task', models.CharField(max_length=100, null=True)), + ('is_image', models.BooleanField(default=False)), + ('image_size', models.IntegerField(default=0)), + ('license_manifest_path', models.CharField(max_length=500, null=True)), + ('build', models.ForeignKey(to='orm.Build')), + ], + ), + migrations.CreateModel( + name='Target_File', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('path', models.FilePathField()), + ('size', models.IntegerField()), + ('inodetype', models.IntegerField(choices=[(1, b'regular'), (2, b'directory'), (3, b'symlink'), (4, b'socket'), (5, b'fifo'), (6, b'character'), (7, b'block')])), + ('permission', models.CharField(max_length=16)), + ('owner', models.CharField(max_length=128)), + ('group', models.CharField(max_length=128)), + ('directory', models.ForeignKey(related_name='directory_set', to='orm.Target_File', null=True)), + ('sym_target', models.ForeignKey(related_name='symlink_set', to='orm.Target_File', null=True)), + ('target', models.ForeignKey(to='orm.Target')), + ], + ), + migrations.CreateModel( + name='Target_Image_File', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('file_name', models.FilePathField(max_length=254)), + ('file_size', models.IntegerField()), + ('target', models.ForeignKey(to='orm.Target')), + ], + ), + migrations.CreateModel( + name='Target_Installed_Package', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('package', models.ForeignKey(related_name='buildtargetlist_package', to='orm.Package')), + ('target', models.ForeignKey(to='orm.Target')), + ], + ), + migrations.CreateModel( + name='Task', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('order', models.IntegerField(null=True)), + ('task_executed', models.BooleanField(default=False)), + ('outcome', models.IntegerField(default=-1, choices=[(-1, b'Not Available'), (0, b'Succeeded'), (1, b'Covered'), (2, b'Cached'), (3, b'Prebuilt'), (4, b'Failed'), (5, b'Empty')])), + ('sstate_checksum', models.CharField(max_length=100, blank=True)), + ('path_to_sstate_obj', models.FilePathField(max_length=500, blank=True)), + ('task_name', models.CharField(max_length=100)), + ('source_url', models.FilePathField(max_length=255, blank=True)), + ('work_directory', models.FilePathField(max_length=255, blank=True)), + ('script_type', models.IntegerField(default=0, choices=[(0, b'N/A'), (2, b'Python'), (3, b'Shell')])), + ('line_number', models.IntegerField(default=0)), + ('disk_io', models.IntegerField(null=True)), + ('cpu_usage', models.DecimalField(null=True, max_digits=8, decimal_places=2)), + ('elapsed_time', models.DecimalField(null=True, max_digits=8, decimal_places=2)), + ('sstate_result', models.IntegerField(default=0, choices=[(0, b'Not Applicable'), (1, b'File not in cache'), (2, b'Failed'), (3, b'Succeeded')])), + ('message', models.CharField(max_length=240)), + ('logfile', models.FilePathField(max_length=255, blank=True)), + ('build', models.ForeignKey(related_name='task_build', to='orm.Build')), + ('recipe', models.ForeignKey(related_name='tasks', to='orm.Recipe')), + ], + options={ + 'ordering': ('order', 'recipe'), + }, + ), + migrations.CreateModel( + name='Task_Dependency', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('depends_on', models.ForeignKey(related_name='task_dependencies_depends', to='orm.Task')), + ('task', models.ForeignKey(related_name='task_dependencies_task', to='orm.Task')), + ], + ), + migrations.CreateModel( + name='ToasterSetting', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=63)), + ('helptext', models.TextField()), + ('value', models.CharField(max_length=255)), + ], + ), + migrations.CreateModel( + name='Variable', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('variable_name', models.CharField(max_length=100)), + ('variable_value', models.TextField(blank=True)), + ('changed', models.BooleanField(default=False)), + ('human_readable_name', models.CharField(max_length=200)), + ('description', models.TextField(blank=True)), + ('build', models.ForeignKey(related_name='variable_build', to='orm.Build')), + ], + ), + migrations.CreateModel( + name='VariableHistory', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('value', models.TextField(blank=True)), + ('file_name', models.FilePathField(max_length=255)), + ('line_number', models.IntegerField(null=True)), + ('operation', models.CharField(max_length=64)), + ('variable', models.ForeignKey(related_name='vhistory', to='orm.Variable')), + ], + ), + migrations.AddField( + model_name='project', + name='release', + field=models.ForeignKey(to='orm.Release', null=True), + ), + migrations.AddField( + model_name='package_dependency', + name='target', + field=models.ForeignKey(to='orm.Target', null=True), + ), + migrations.AddField( + model_name='package', + name='recipe', + field=models.ForeignKey(to='orm.Recipe', null=True), + ), + migrations.AddField( + model_name='logmessage', + name='task', + field=models.ForeignKey(blank=True, to='orm.Task', null=True), + ), + migrations.AlterUniqueTogether( + name='layersource', + unique_together=set([('sourcetype', 'apiurl')]), + ), + migrations.AddField( + model_name='layer_version', + name='layer_source', + field=models.ForeignKey(default=None, to='orm.LayerSource', null=True), + ), + migrations.AddField( + model_name='layer_version', + name='project', + field=models.ForeignKey(default=None, to='orm.Project', null=True), + ), + migrations.AddField( + model_name='layer_version', + name='up_branch', + field=models.ForeignKey(default=None, to='orm.Branch', null=True), + ), + migrations.AddField( + model_name='layer', + name='layer_source', + field=models.ForeignKey(default=None, to='orm.LayerSource', null=True), + ), + migrations.AddField( + model_name='build', + name='project', + field=models.ForeignKey(to='orm.Project'), + ), + migrations.AddField( + model_name='branch', + name='layer_source', + field=models.ForeignKey(default=True, to='orm.LayerSource', null=True), + ), + migrations.CreateModel( + name='ImportedLayerSource', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('orm.layersource',), + ), + migrations.CreateModel( + name='LayerIndexLayerSource', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('orm.layersource',), + ), + migrations.CreateModel( + name='LocalLayerSource', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('orm.layersource',), + ), + migrations.AlterUniqueTogether( + name='task', + unique_together=set([('build', 'recipe', 'task_name')]), + ), + migrations.AlterUniqueTogether( + name='releaselayersourcepriority', + unique_together=set([('release', 'layer_source')]), + ), + migrations.AlterUniqueTogether( + name='recipe', + unique_together=set([('layer_version', 'file_path', 'pathflags')]), + ), + migrations.AlterUniqueTogether( + name='projectlayer', + unique_together=set([('project', 'layercommit')]), + ), + migrations.AlterUniqueTogether( + name='machine', + unique_together=set([('layer_source', 'up_id')]), + ), + migrations.AlterUniqueTogether( + name='layerversiondependency', + unique_together=set([('layer_source', 'up_id')]), + ), + migrations.AlterUniqueTogether( + name='layer_version', + unique_together=set([('layer_source', 'up_id')]), + ), + migrations.AlterUniqueTogether( + name='layer', + unique_together=set([('layer_source', 'up_id'), ('layer_source', 'name')]), + ), + migrations.AlterUniqueTogether( + name='branch', + unique_together=set([('layer_source', 'up_id'), ('layer_source', 'name')]), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0002_customimagerecipe.py b/poky/bitbake/lib/toaster/orm/migrations/0002_customimagerecipe.py new file mode 100644 index 000000000..9cec82e8d --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0002_customimagerecipe.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='CustomImageRecipe', + fields=[ + ('recipe_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='orm.Recipe')), + ('last_updated', models.DateTimeField(default=None, null=True)), + ('base_recipe', models.ForeignKey(related_name='based_on_recipe', to='orm.Recipe')), + ('project', models.ForeignKey(to='orm.Project')), + ], + bases=('orm.recipe',), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0003_customimagepackage.py b/poky/bitbake/lib/toaster/orm/migrations/0003_customimagepackage.py new file mode 100644 index 000000000..b027f6613 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0003_customimagepackage.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0002_customimagerecipe'), + ] + + operations = [ + migrations.CreateModel( + name='CustomImagePackage', + fields=[ + ('package_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='orm.Package')), + ('recipe_appends', models.ManyToManyField(related_name='appends_set', to='orm.CustomImageRecipe')), + ('recipe_excludes', models.ManyToManyField(related_name='excludes_set', to='orm.CustomImageRecipe')), + ('recipe_includes', models.ManyToManyField(related_name='includes_set', to='orm.CustomImageRecipe')), + ], + bases=('orm.package',), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0004_provides.py b/poky/bitbake/lib/toaster/orm/migrations/0004_provides.py new file mode 100644 index 000000000..dfde2d136 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0004_provides.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0003_customimagepackage'), + ] + + operations = [ + migrations.CreateModel( + name='Provides', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=100)), + ('recipe', models.ForeignKey(to='orm.Recipe')), + ], + ), + migrations.AddField( + model_name='recipe_dependency', + name='via', + field=models.ForeignKey(null=True, default=None, to='orm.Provides'), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py b/poky/bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py new file mode 100644 index 000000000..fb1196b56 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0004_provides'), + ] + + operations = [ + migrations.RemoveField( + model_name='task', + name='cpu_usage', + ), + migrations.AddField( + model_name='task', + name='cpu_time_system', + field=models.DecimalField(null=True, max_digits=8, decimal_places=2), + ), + migrations.AddField( + model_name='task', + name='cpu_time_user', + field=models.DecimalField(null=True, max_digits=8, decimal_places=2), + ), + migrations.AddField( + model_name='task', + name='disk_io_read', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='task', + name='disk_io_write', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='task', + name='ended', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='task', + name='started', + field=models.DateTimeField(null=True), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0006_add_cancelled_state.py b/poky/bitbake/lib/toaster/orm/migrations/0006_add_cancelled_state.py new file mode 100644 index 000000000..91a32a9e0 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0006_add_cancelled_state.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0005_task_field_separation'), + ] + + operations = [ + migrations.AlterField( + model_name='build', + name='outcome', + field=models.IntegerField(default=2, choices=[(0, b'Succeeded'), (1, b'Failed'), (2, b'In Progress'), (3, b'Cancelled')]), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0007_auto_20160523_1446.py b/poky/bitbake/lib/toaster/orm/migrations/0007_auto_20160523_1446.py new file mode 100644 index 000000000..b472e7cf0 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0007_auto_20160523_1446.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0006_add_cancelled_state'), + ] + + operations = [ + migrations.AlterField( + model_name='build', + name='outcome', + field=models.IntegerField(default=2, choices=[(0, 'Succeeded'), (1, 'Failed'), (2, 'In Progress'), (3, 'Cancelled')]), + ), + migrations.AlterField( + model_name='helptext', + name='area', + field=models.IntegerField(choices=[(0, 'variable')]), + ), + migrations.AlterField( + model_name='layer', + name='summary', + field=models.TextField(default=None, null=True, help_text='One-line description of the layer'), + ), + migrations.AlterField( + model_name='layer_version', + name='local_path', + field=models.FilePathField(default='/', max_length=1024), + ), + migrations.AlterField( + model_name='layersource', + name='sourcetype', + field=models.IntegerField(choices=[(0, 'local'), (1, 'layerindex'), (2, 'imported')]), + ), + migrations.AlterField( + model_name='logmessage', + name='level', + field=models.IntegerField(default=0, choices=[(0, 'info'), (1, 'warn'), (2, 'error'), (3, 'critical'), (-1, 'toaster exception')]), + ), + migrations.AlterField( + model_name='package', + name='installed_name', + field=models.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='package_dependency', + name='dep_type', + field=models.IntegerField(choices=[(0, 'depends'), (1, 'depends'), (3, 'recommends'), (2, 'recommends'), (4, 'suggests'), (5, 'provides'), (6, 'replaces'), (7, 'conflicts')]), + ), + migrations.AlterField( + model_name='recipe_dependency', + name='dep_type', + field=models.IntegerField(choices=[(0, 'depends'), (1, 'rdepends')]), + ), + migrations.AlterField( + model_name='release', + name='branch_name', + field=models.CharField(default='', max_length=50), + ), + migrations.AlterField( + model_name='releasedefaultlayer', + name='layer_name', + field=models.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='target_file', + name='inodetype', + field=models.IntegerField(choices=[(1, 'regular'), (2, 'directory'), (3, 'symlink'), (4, 'socket'), (5, 'fifo'), (6, 'character'), (7, 'block')]), + ), + migrations.AlterField( + model_name='task', + name='outcome', + field=models.IntegerField(default=-1, choices=[(-1, 'Not Available'), (0, 'Succeeded'), (1, 'Covered'), (2, 'Cached'), (3, 'Prebuilt'), (4, 'Failed'), (5, 'Empty')]), + ), + migrations.AlterField( + model_name='task', + name='script_type', + field=models.IntegerField(default=0, choices=[(0, 'N/A'), (2, 'Python'), (3, 'Shell')]), + ), + migrations.AlterField( + model_name='task', + name='sstate_result', + field=models.IntegerField(default=0, choices=[(0, 'Not Applicable'), (1, 'File not in cache'), (2, 'Failed'), (3, 'Succeeded')]), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py b/poky/bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py new file mode 100644 index 000000000..3367582a8 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0007_auto_20160523_1446'), + ] + + operations = [ + migrations.CreateModel( + name='TargetKernelFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)), + ('file_name', models.FilePathField()), + ('file_size', models.IntegerField()), + ('target', models.ForeignKey(to='orm.Target')), + ], + ), + migrations.CreateModel( + name='TargetSDKFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)), + ('file_name', models.FilePathField()), + ('file_size', models.IntegerField()), + ('target', models.ForeignKey(to='orm.Target')), + ], + ), + migrations.RemoveField( + model_name='buildartifact', + name='build', + ), + migrations.DeleteModel( + name='BuildArtifact', + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0009_target_package_manifest_path.py b/poky/bitbake/lib/toaster/orm/migrations/0009_target_package_manifest_path.py new file mode 100644 index 000000000..c958f3070 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0009_target_package_manifest_path.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0008_refactor_artifact_models'), + ] + + operations = [ + migrations.AddField( + model_name='target', + name='package_manifest_path', + field=models.CharField(null=True, max_length=500), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0010_delete_layer_source_references.py b/poky/bitbake/lib/toaster/orm/migrations/0010_delete_layer_source_references.py new file mode 100644 index 000000000..f67388e99 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0010_delete_layer_source_references.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0009_target_package_manifest_path'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='releaselayersourcepriority', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='releaselayersourcepriority', + name='layer_source', + ), + migrations.RemoveField( + model_name='releaselayersourcepriority', + name='release', + ), + migrations.DeleteModel( + name='ImportedLayerSource', + ), + migrations.DeleteModel( + name='LayerIndexLayerSource', + ), + migrations.DeleteModel( + name='LocalLayerSource', + ), + migrations.RemoveField( + model_name='recipe', + name='layer_source', + ), + migrations.RemoveField( + model_name='recipe', + name='up_id', + ), + migrations.AlterField( + model_name='layer', + name='up_date', + field=models.DateTimeField(default=django.utils.timezone.now, null=True), + ), + migrations.AlterField( + model_name='layer_version', + name='layer_source', + field=models.IntegerField(default=0, choices=[(0, 'local'), (1, 'layerindex'), (2, 'imported'), (3, 'build')]), + ), + migrations.AlterField( + model_name='layer_version', + name='up_date', + field=models.DateTimeField(default=django.utils.timezone.now, null=True), + ), + migrations.AlterUniqueTogether( + name='branch', + unique_together=set([]), + ), + migrations.AlterUniqueTogether( + name='layer', + unique_together=set([]), + ), + migrations.AlterUniqueTogether( + name='layer_version', + unique_together=set([]), + ), + migrations.AlterUniqueTogether( + name='layerversiondependency', + unique_together=set([]), + ), + migrations.AlterUniqueTogether( + name='machine', + unique_together=set([]), + ), + migrations.DeleteModel( + name='ReleaseLayerSourcePriority', + ), + migrations.RemoveField( + model_name='branch', + name='layer_source', + ), + migrations.RemoveField( + model_name='branch', + name='up_id', + ), + migrations.RemoveField( + model_name='layer', + name='layer_source', + ), + migrations.RemoveField( + model_name='layer', + name='up_id', + ), + migrations.RemoveField( + model_name='layer_version', + name='up_id', + ), + migrations.RemoveField( + model_name='layerversiondependency', + name='layer_source', + ), + migrations.RemoveField( + model_name='layerversiondependency', + name='up_id', + ), + migrations.RemoveField( + model_name='machine', + name='layer_source', + ), + migrations.RemoveField( + model_name='machine', + name='up_id', + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0011_delete_layersource.py b/poky/bitbake/lib/toaster/orm/migrations/0011_delete_layersource.py new file mode 100644 index 000000000..75506961a --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0011_delete_layersource.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0010_delete_layer_source_references'), + ] + + operations = [ + migrations.DeleteModel( + name='LayerSource', + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0012_use_release_instead_of_up_branch.py b/poky/bitbake/lib/toaster/orm/migrations/0012_use_release_instead_of_up_branch.py new file mode 100644 index 000000000..0e6bb8331 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0012_use_release_instead_of_up_branch.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +from django.db.models import Q + + +def branch_to_release(apps, schema_editor): + Layer_Version = apps.get_model('orm', 'Layer_Version') + Release = apps.get_model('orm', 'Release') + + print("Converting all layer version up_branches to releases") + # Find all the layer versions which have an upbranch and convert them to + # the release that they're for. + for layer_version in Layer_Version.objects.filter( + Q(release=None) & ~Q(up_branch=None)): + try: + # HEAD and local are equivalent + if "HEAD" in layer_version.up_branch.name: + release = Release.objects.get(name="local") + layer_version.commit = "HEAD" + layer_version.branch = "HEAD" + else: + release = Release.objects.get( + name=layer_version.up_branch.name) + + layer_version.release = release + layer_version.save() + except Exception as e: + print("Couldn't work out an appropriate release for %s " + "the up_branch was %s " + "user the django admin interface to correct it" % + (layer_version.layer.name, layer_version.up_branch.name)) + print(e) + + continue + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0011_delete_layersource'), + ] + + operations = [ + migrations.AddField( + model_name='layer_version', + name='release', + field=models.ForeignKey(to='orm.Release', default=None, null=True), + ), + migrations.RunPython(branch_to_release, + reverse_code=migrations.RunPython.noop), + + migrations.RemoveField( + model_name='layer_version', + name='up_branch', + ), + + migrations.DeleteModel( + name='Branch', + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py b/poky/bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py new file mode 100644 index 000000000..cc5c96d2d --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0012_use_release_instead_of_up_branch'), + ] + + operations = [ + migrations.AddField( + model_name='build', + name='recipes_parsed', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='build', + name='recipes_to_parse', + field=models.IntegerField(default=1), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py b/poky/bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py new file mode 100644 index 000000000..4749a14b2 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0013_recipe_parse_progress_fields'), + ] + + operations = [ + migrations.AlterField( + model_name='build', + name='build_name', + field=models.CharField(default='', max_length=100), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0015_layer_local_source_dir.py b/poky/bitbake/lib/toaster/orm/migrations/0015_layer_local_source_dir.py new file mode 100644 index 000000000..9539cd72a --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0015_layer_local_source_dir.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0014_allow_empty_buildname'), + ] + + operations = [ + migrations.AddField( + model_name='layer', + name='local_source_dir', + field=models.TextField(null=True, default=None), + ), + ] diff --git a/poky/bitbake/lib/toaster/orm/migrations/0016_clone_progress.py b/poky/bitbake/lib/toaster/orm/migrations/0016_clone_progress.py new file mode 100644 index 000000000..cd4023b6f --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0016_clone_progress.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0015_layer_local_source_dir'), + ] + + operations = [ + migrations.AddField( + model_name='build', + name='repos_cloned', + field=models.IntegerField(default=1), + ), + migrations.AddField( + model_name='build', + name='repos_to_clone', + field=models.IntegerField(default=1), # (default off) + ), + ] + diff --git a/poky/bitbake/lib/toaster/orm/migrations/0017_distro_clone.py b/poky/bitbake/lib/toaster/orm/migrations/0017_distro_clone.py new file mode 100644 index 000000000..d3c590127 --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/migrations/0017_distro_clone.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0016_clone_progress'), + ] + + operations = [ + migrations.CreateModel( + name='Distro', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('up_id', models.IntegerField(default=None, null=True)), + ('up_date', models.DateTimeField(default=None, null=True)), + ('name', models.CharField(max_length=255)), + ('description', models.CharField(max_length=255)), + ('layer_version', models.ForeignKey(to='orm.Layer_Version')), + ], + ), + ] + diff --git a/poky/bitbake/lib/toaster/orm/migrations/__init__.py b/poky/bitbake/lib/toaster/orm/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/orm/models.py b/poky/bitbake/lib/toaster/orm/models.py new file mode 100644 index 000000000..3a7dff8ca --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/models.py @@ -0,0 +1,1832 @@ +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import unicode_literals + +from django.db import models, IntegrityError, DataError +from django.db.models import F, Q, Sum, Count +from django.utils import timezone +from django.utils.encoding import force_bytes + +from django.core.urlresolvers import reverse + +from django.core import validators +from django.conf import settings +import django.db.models.signals + +import sys +import os +import re +import itertools +from signal import SIGUSR1 + + +import logging +logger = logging.getLogger("toaster") + +if 'sqlite' in settings.DATABASES['default']['ENGINE']: + from django.db import transaction, OperationalError + from time import sleep + + _base_save = models.Model.save + def save(self, *args, **kwargs): + while True: + try: + with transaction.atomic(): + return _base_save(self, *args, **kwargs) + except OperationalError as err: + if 'database is locked' in str(err): + logger.warning("%s, model: %s, args: %s, kwargs: %s", + err, self.__class__, args, kwargs) + sleep(0.5) + continue + raise + + models.Model.save = save + + # HACK: Monkey patch Django to fix 'database is locked' issue + + from django.db.models.query import QuerySet + _base_insert = QuerySet._insert + def _insert(self, *args, **kwargs): + with transaction.atomic(using=self.db, savepoint=False): + return _base_insert(self, *args, **kwargs) + QuerySet._insert = _insert + + from django.utils import six + def _create_object_from_params(self, lookup, params): + """ + Tries to create an object using passed params. + Used by get_or_create and update_or_create + """ + try: + obj = self.create(**params) + return obj, True + except (IntegrityError, DataError): + exc_info = sys.exc_info() + try: + return self.get(**lookup), False + except self.model.DoesNotExist: + pass + six.reraise(*exc_info) + + QuerySet._create_object_from_params = _create_object_from_params + + # end of HACK + +class GitURLValidator(validators.URLValidator): + import re + regex = re.compile( + r'^(?:ssh|git|http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4 + r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6 + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + +def GitURLField(**kwargs): + r = models.URLField(**kwargs) + for i in range(len(r.validators)): + if isinstance(r.validators[i], validators.URLValidator): + r.validators[i] = GitURLValidator() + return r + + +class ToasterSetting(models.Model): + name = models.CharField(max_length=63) + helptext = models.TextField() + value = models.CharField(max_length=255) + + def __unicode__(self): + return "Setting %s = %s" % (self.name, self.value) + + +class ProjectManager(models.Manager): + def create_project(self, name, release): + if release is not None: + prj = self.model(name=name, + bitbake_version=release.bitbake_version, + release=release) + else: + prj = self.model(name=name, + bitbake_version=None, + release=None) + + prj.save() + + for defaultconf in ToasterSetting.objects.filter( + name__startswith="DEFCONF_"): + name = defaultconf.name[8:] + ProjectVariable.objects.create(project=prj, + name=name, + value=defaultconf.value) + + if release is None: + return prj + + for rdl in release.releasedefaultlayer_set.all(): + lv = Layer_Version.objects.filter( + layer__name=rdl.layer_name, + release=release).first() + + if lv: + ProjectLayer.objects.create(project=prj, + layercommit=lv, + optional=False) + else: + logger.warning("Default project layer %s not found" % + rdl.layer_name) + + return prj + + # return single object with is_default = True + def get_or_create_default_project(self): + projects = super(ProjectManager, self).filter(is_default=True) + + if len(projects) > 1: + raise Exception('Inconsistent project data: multiple ' + + 'default projects (i.e. with is_default=True)') + elif len(projects) < 1: + options = { + 'name': 'Command line builds', + 'short_description': + 'Project for builds started outside Toaster', + 'is_default': True + } + project = Project.objects.create(**options) + project.save() + + return project + else: + return projects[0] + + +class Project(models.Model): + search_allowed_fields = ['name', 'short_description', 'release__name', + 'release__branch_name'] + name = models.CharField(max_length=100) + short_description = models.CharField(max_length=50, blank=True) + bitbake_version = models.ForeignKey('BitbakeVersion', null=True) + release = models.ForeignKey("Release", null=True) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + # This is a horrible hack; since Toaster has no "User" model available when + # running in interactive mode, we can't reference the field here directly + # Instead, we keep a possible null reference to the User id, + # as not to force + # hard links to possibly missing models + user_id = models.IntegerField(null=True) + objects = ProjectManager() + + # set to True for the project which is the default container + # for builds initiated by the command line etc. + is_default= models.BooleanField(default=False) + + def __unicode__(self): + return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version) + + def get_current_machine_name(self): + try: + return self.projectvariable_set.get(name="MACHINE").value + except (ProjectVariable.DoesNotExist,IndexError): + return None; + + def get_number_of_builds(self): + """Return the number of builds which have ended""" + + return self.build_set.exclude( + Q(outcome=Build.IN_PROGRESS) | + Q(outcome=Build.CANCELLED) + ).count() + + def get_last_build_id(self): + try: + return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id + except (Build.DoesNotExist,IndexError): + return( -1 ) + + def get_last_outcome(self): + build_id = self.get_last_build_id() + if (-1 == build_id): + return( "" ) + try: + return Build.objects.filter( id = build_id )[ 0 ].outcome + except (Build.DoesNotExist,IndexError): + return( "not_found" ) + + def get_last_target(self): + build_id = self.get_last_build_id() + if (-1 == build_id): + return( "" ) + try: + return Target.objects.filter(build = build_id)[0].target + except (Target.DoesNotExist,IndexError): + return( "not_found" ) + + def get_last_errors(self): + build_id = self.get_last_build_id() + if (-1 == build_id): + return( 0 ) + try: + return Build.objects.filter(id = build_id)[ 0 ].errors.count() + except (Build.DoesNotExist,IndexError): + return( "not_found" ) + + def get_last_warnings(self): + build_id = self.get_last_build_id() + if (-1 == build_id): + return( 0 ) + try: + return Build.objects.filter(id = build_id)[ 0 ].warnings.count() + except (Build.DoesNotExist,IndexError): + return( "not_found" ) + + def get_last_build_extensions(self): + """ + Get list of file name extensions for images produced by the most + recent build + """ + last_build = Build.objects.get(pk = self.get_last_build_id()) + return last_build.get_image_file_extensions() + + def get_last_imgfiles(self): + build_id = self.get_last_build_id() + if (-1 == build_id): + return( "" ) + try: + return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value + except (Variable.DoesNotExist,IndexError): + return( "not_found" ) + + def get_all_compatible_layer_versions(self): + """ Returns Queryset of all Layer_Versions which are compatible with + this project""" + queryset = None + + # guard on release, as it can be null + if self.release: + queryset = Layer_Version.objects.filter( + (Q(release=self.release) & + Q(build=None) & + Q(project=None)) | + Q(project=self)) + else: + queryset = Layer_Version.objects.none() + + return queryset + + def get_project_layer_versions(self, pk=False): + """ Returns the Layer_Versions currently added to this project """ + layer_versions = self.projectlayer_set.all().values_list('layercommit', + flat=True) + + if pk is False: + return Layer_Version.objects.filter(pk__in=layer_versions) + else: + return layer_versions + + + def get_available_machines(self): + """ Returns QuerySet of all Machines which are provided by the + Layers currently added to the Project """ + queryset = Machine.objects.filter( + layer_version__in=self.get_project_layer_versions()) + + return queryset + + def get_all_compatible_machines(self): + """ Returns QuerySet of all the compatible machines available to the + project including ones from Layers not currently added """ + queryset = Machine.objects.filter( + layer_version__in=self.get_all_compatible_layer_versions()) + + return queryset + + def get_available_distros(self): + """ Returns QuerySet of all Distros which are provided by the + Layers currently added to the Project """ + queryset = Distro.objects.filter( + layer_version__in=self.get_project_layer_versions()) + + return queryset + + def get_all_compatible_distros(self): + """ Returns QuerySet of all the compatible Wind River distros available to the + project including ones from Layers not currently added """ + queryset = Distro.objects.filter( + layer_version__in=self.get_all_compatible_layer_versions()) + + return queryset + + def get_available_recipes(self): + """ Returns QuerySet of all the recipes that are provided by layers + added to this project """ + queryset = Recipe.objects.filter( + layer_version__in=self.get_project_layer_versions()) + + return queryset + + def get_all_compatible_recipes(self): + """ Returns QuerySet of all the compatible Recipes available to the + project including ones from Layers not currently added """ + queryset = Recipe.objects.filter( + layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='') + + return queryset + + def schedule_build(self): + + from bldcontrol.models import BuildRequest, BRTarget, BRLayer + from bldcontrol.models import BRBitbake, BRVariable + + try: + now = timezone.now() + build = Build.objects.create(project=self, + completed_on=now, + started_on=now) + + br = BuildRequest.objects.create(project=self, + state=BuildRequest.REQ_QUEUED, + build=build) + BRBitbake.objects.create(req=br, + giturl=self.bitbake_version.giturl, + commit=self.bitbake_version.branch, + dirpath=self.bitbake_version.dirpath) + + for t in self.projecttarget_set.all(): + BRTarget.objects.create(req=br, target=t.target, task=t.task) + Target.objects.create(build=br.build, target=t.target, + task=t.task) + # If we're about to build a custom image recipe make sure + # that layer is currently in the project before we create the + # BRLayer objects + customrecipe = CustomImageRecipe.objects.filter( + name=t.target, + project=self).first() + if customrecipe: + ProjectLayer.objects.get_or_create( + project=self, + layercommit=customrecipe.layer_version, + optional=False) + + for l in self.projectlayer_set.all().order_by("pk"): + commit = l.layercommit.get_vcs_reference() + logger.debug("Adding layer to build %s" % + l.layercommit.layer.name) + BRLayer.objects.create( + req=br, + name=l.layercommit.layer.name, + giturl=l.layercommit.layer.vcs_url, + commit=commit, + dirpath=l.layercommit.dirpath, + layer_version=l.layercommit, + local_source_dir=l.layercommit.layer.local_source_dir + ) + + for v in self.projectvariable_set.all(): + BRVariable.objects.create(req=br, name=v.name, value=v.value) + + try: + br.build.machine = self.projectvariable_set.get( + name='MACHINE').value + br.build.save() + except ProjectVariable.DoesNotExist: + pass + + br.save() + signal_runbuilds() + + except Exception: + # revert the build request creation since we're not done cleanly + br.delete() + raise + return br + +class Build(models.Model): + SUCCEEDED = 0 + FAILED = 1 + IN_PROGRESS = 2 + CANCELLED = 3 + + BUILD_OUTCOME = ( + (SUCCEEDED, 'Succeeded'), + (FAILED, 'Failed'), + (IN_PROGRESS, 'In Progress'), + (CANCELLED, 'Cancelled'), + ) + + search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"] + + project = models.ForeignKey(Project) # must have a project + machine = models.CharField(max_length=100) + distro = models.CharField(max_length=100) + distro_version = models.CharField(max_length=100) + started_on = models.DateTimeField() + completed_on = models.DateTimeField() + outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS) + cooker_log_path = models.CharField(max_length=500) + build_name = models.CharField(max_length=100, default='') + bitbake_version = models.CharField(max_length=50) + + # number of recipes to parse for this build + recipes_to_parse = models.IntegerField(default=1) + + # number of recipes parsed so far for this build + recipes_parsed = models.IntegerField(default=1) + + # number of repos to clone for this build + repos_to_clone = models.IntegerField(default=1) + + # number of repos cloned so far for this build (default off) + repos_cloned = models.IntegerField(default=1) + + @staticmethod + def get_recent(project=None): + """ + Return recent builds as a list; if project is set, only return + builds for that project + """ + + builds = Build.objects.all() + + if project: + builds = builds.filter(project=project) + + finished_criteria = \ + Q(outcome=Build.SUCCEEDED) | \ + Q(outcome=Build.FAILED) | \ + Q(outcome=Build.CANCELLED) + + recent_builds = list(itertools.chain( + builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"), + builds.filter(finished_criteria).order_by("-completed_on")[:3] + )) + + # add percentage done property to each build; this is used + # to show build progress in mrb_section.html + for build in recent_builds: + build.percentDone = build.completeper() + build.outcomeText = build.get_outcome_text() + + return recent_builds + + def started(self): + """ + As build variables are only added for a build when its BuildStarted event + is received, a build with no build variables is counted as + "in preparation" and not properly started yet. This method + will return False if a build has no build variables (it never properly + started), or True otherwise. + + Note that this is a temporary workaround for the fact that we don't + have a fine-grained state variable on a build which would allow us + to record "in progress" (BuildStarted received) vs. "in preparation". + """ + variables = Variable.objects.filter(build=self) + return len(variables) > 0 + + def completeper(self): + tf = Task.objects.filter(build = self) + tfc = tf.count() + if tfc > 0: + completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc + else: + completeper = 0 + return completeper + + def eta(self): + eta = timezone.now() + completeper = self.completeper() + if self.completeper() > 0: + eta += ((eta - self.started_on)*(100-completeper))/completeper + return eta + + def has_images(self): + """ + Returns True if at least one of the targets for this build has an + image file associated with it, False otherwise + """ + targets = Target.objects.filter(build_id=self.id) + has_images = False + for target in targets: + if target.has_images(): + has_images = True + break + return has_images + + def has_image_recipes(self): + """ + Returns True if a build has any targets which were built from + image recipes. + """ + image_recipes = self.get_image_recipes() + return len(image_recipes) > 0 + + def get_image_file_extensions(self): + """ + Get string of file name extensions for images produced by this build; + note that this is the actual list of extensions stored on Target objects + for this build, and not the value of IMAGE_FSTYPES. + + Returns comma-separated string, e.g. "vmdk, ext4" + """ + extensions = [] + + targets = Target.objects.filter(build_id = self.id) + for target in targets: + if not target.is_image: + continue + + target_image_files = Target_Image_File.objects.filter( + target_id=target.id) + + for target_image_file in target_image_files: + extensions.append(target_image_file.suffix) + + extensions = list(set(extensions)) + extensions.sort() + + return ', '.join(extensions) + + def get_image_fstypes(self): + """ + Get the IMAGE_FSTYPES variable value for this build as a de-duplicated + list of image file suffixes. + """ + image_fstypes = Variable.objects.get( + build=self, variable_name='IMAGE_FSTYPES').variable_value + return list(set(re.split(r' {1,}', image_fstypes))) + + def get_sorted_target_list(self): + tgts = Target.objects.filter(build_id = self.id).order_by( 'target' ); + return( tgts ); + + def get_recipes(self): + """ + Get the recipes related to this build; + note that the related layer versions and layers are also prefetched + by this query, as this queryset can be sorted by these objects in the + build recipes view; prefetching them here removes the need + for another query in that view + """ + layer_versions = Layer_Version.objects.filter(build=self) + criteria = Q(layer_version__id__in=layer_versions) + return Recipe.objects.filter(criteria) \ + .select_related('layer_version', 'layer_version__layer') + + def get_image_recipes(self): + """ + Returns a list of image Recipes (custom and built-in) related to this + build, sorted by name; note that this has to be done in two steps, as + there's no way to get all the custom image recipes and image recipes + in one query + """ + custom_image_recipes = self.get_custom_image_recipes() + custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True) + + not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \ + Q(is_image=True) + + built_image_recipes = self.get_recipes().filter(not_custom_image_recipes) + + # append to the custom image recipes and sort + customisable_image_recipes = list( + itertools.chain(custom_image_recipes, built_image_recipes) + ) + + return sorted(customisable_image_recipes, key=lambda recipe: recipe.name) + + def get_custom_image_recipes(self): + """ + Returns a queryset of CustomImageRecipes related to this build, + sorted by name + """ + built_recipe_names = self.get_recipes().values_list('name', flat=True) + criteria = Q(name__in=built_recipe_names) & Q(project=self.project) + queryset = CustomImageRecipe.objects.filter(criteria).order_by('name') + return queryset + + def get_outcome_text(self): + return Build.BUILD_OUTCOME[int(self.outcome)][1] + + @property + def failed_tasks(self): + """ Get failed tasks for the build """ + tasks = self.task_build.all() + return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED) + + @property + def errors(self): + return (self.logmessage_set.filter(level=LogMessage.ERROR) | + self.logmessage_set.filter(level=LogMessage.EXCEPTION) | + self.logmessage_set.filter(level=LogMessage.CRITICAL)) + + @property + def warnings(self): + return self.logmessage_set.filter(level=LogMessage.WARNING) + + @property + def timespent(self): + return self.completed_on - self.started_on + + @property + def timespent_seconds(self): + return self.timespent.total_seconds() + + @property + def target_labels(self): + """ + Sorted (a-z) "target1:task, target2, target3" etc. string for all + targets in this build + """ + targets = self.target_set.all() + target_labels = [target.target + + (':' + target.task if target.task else '') + for target in targets] + target_labels.sort() + + return target_labels + + def get_buildrequest(self): + buildrequest = None + if hasattr(self, 'buildrequest'): + buildrequest = self.buildrequest + return buildrequest + + def is_queued(self): + from bldcontrol.models import BuildRequest + buildrequest = self.get_buildrequest() + if buildrequest: + return buildrequest.state == BuildRequest.REQ_QUEUED + else: + return False + + def is_cancelling(self): + from bldcontrol.models import BuildRequest + buildrequest = self.get_buildrequest() + if buildrequest: + return self.outcome == Build.IN_PROGRESS and \ + buildrequest.state == BuildRequest.REQ_CANCELLING + else: + return False + + def is_cloning(self): + """ + True if the build is still cloning repos + """ + return self.outcome == Build.IN_PROGRESS and \ + self.repos_cloned < self.repos_to_clone + + def is_parsing(self): + """ + True if the build is still parsing recipes + """ + return self.outcome == Build.IN_PROGRESS and \ + self.recipes_parsed < self.recipes_to_parse + + def is_starting(self): + """ + True if the build has no completed tasks yet and is still just starting + tasks. + + Note that the mechanism for testing whether a Task is "done" is whether + its outcome field is set, as per the completeper() method. + """ + return self.outcome == Build.IN_PROGRESS and \ + self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0 + + + def get_state(self): + """ + Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress', + 'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states + dependent on the BuildRequest state). + + This works around the fact that we have BuildRequest states as well + as Build states, but really we just want to know the state of the build. + """ + if self.is_cancelling(): + return 'Cancelling'; + elif self.is_queued(): + return 'Queued' + elif self.is_cloning(): + return 'Cloning' + elif self.is_parsing(): + return 'Parsing' + elif self.is_starting(): + return 'Starting' + else: + return self.get_outcome_text() + + def __str__(self): + return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()])) + +class ProjectTarget(models.Model): + project = models.ForeignKey(Project) + target = models.CharField(max_length=100) + task = models.CharField(max_length=100, null=True) + +class Target(models.Model): + search_allowed_fields = ['target', 'file_name'] + build = models.ForeignKey(Build) + target = models.CharField(max_length=100) + task = models.CharField(max_length=100, null=True) + is_image = models.BooleanField(default = False) + image_size = models.IntegerField(default=0) + license_manifest_path = models.CharField(max_length=500, null=True) + package_manifest_path = models.CharField(max_length=500, null=True) + + def package_count(self): + return Target_Installed_Package.objects.filter(target_id__exact=self.id).count() + + def __unicode__(self): + return self.target + + def get_similar_targets(self): + """ + Get target sfor the same machine, task and target name + (e.g. 'core-image-minimal') from a successful build for this project + (but excluding this target). + + Note that we only look for targets built by this project because + projects can have different configurations from each other, and put + their artifacts in different directories. + + The possibility of error when retrieving candidate targets + is minimised by the fact that bitbake will rebuild artifacts if MACHINE + (or various other variables) change. In this case, there is no need to + clone artifacts from another target, as those artifacts will have + been re-generated for this target anyway. + """ + query = ~Q(pk=self.pk) & \ + Q(target=self.target) & \ + Q(build__machine=self.build.machine) & \ + Q(build__outcome=Build.SUCCEEDED) & \ + Q(build__project=self.build.project) + + return Target.objects.filter(query) + + def get_similar_target_with_image_files(self): + """ + Get the most recent similar target with Target_Image_Files associated + with it, for the purpose of cloning those files onto this target. + """ + similar_target = None + + candidates = self.get_similar_targets() + if candidates.count() == 0: + return similar_target + + task_subquery = Q(task=self.task) + + # we can look for a 'build' task if this task is a 'populate_sdk_ext' + # task, as the latter also creates images; and vice versa; note that + # 'build' targets can have their task set to ''; + # also note that 'populate_sdk' does not produce image files + image_tasks = [ + '', # aka 'build' + 'build', + 'image', + 'populate_sdk_ext' + ] + if self.task in image_tasks: + task_subquery = Q(task__in=image_tasks) + + # annotate with the count of files, to exclude any targets which + # don't have associated files + candidates = candidates.annotate(num_files=Count('target_image_file')) + + query = task_subquery & Q(num_files__gt=0) + + candidates = candidates.filter(query) + + if candidates.count() > 0: + candidates.order_by('build__completed_on') + similar_target = candidates.last() + + return similar_target + + def get_similar_target_with_sdk_files(self): + """ + Get the most recent similar target with TargetSDKFiles associated + with it, for the purpose of cloning those files onto this target. + """ + similar_target = None + + candidates = self.get_similar_targets() + if candidates.count() == 0: + return similar_target + + # annotate with the count of files, to exclude any targets which + # don't have associated files + candidates = candidates.annotate(num_files=Count('targetsdkfile')) + + query = Q(task=self.task) & Q(num_files__gt=0) + + candidates = candidates.filter(query) + + if candidates.count() > 0: + candidates.order_by('build__completed_on') + similar_target = candidates.last() + + return similar_target + + def clone_image_artifacts_from(self, target): + """ + Make clones of the Target_Image_Files and TargetKernelFile objects + associated with Target target, then associate them with this target. + + Note that for Target_Image_Files, we only want files from the previous + build whose suffix matches one of the suffixes defined in this + target's build's IMAGE_FSTYPES configuration variable. This prevents the + Target_Image_File object for an ext4 image being associated with a + target for a project which didn't produce an ext4 image (for example). + + Also sets the license_manifest_path and package_manifest_path + of this target to the same path as that of target being cloned from, as + the manifests are also build artifacts but are treated differently. + """ + + image_fstypes = self.build.get_image_fstypes() + + # filter out any image files whose suffixes aren't in the + # IMAGE_FSTYPES suffixes variable for this target's build + image_files = [target_image_file \ + for target_image_file in target.target_image_file_set.all() \ + if target_image_file.suffix in image_fstypes] + + for image_file in image_files: + image_file.pk = None + image_file.target = self + image_file.save() + + kernel_files = target.targetkernelfile_set.all() + for kernel_file in kernel_files: + kernel_file.pk = None + kernel_file.target = self + kernel_file.save() + + self.license_manifest_path = target.license_manifest_path + self.package_manifest_path = target.package_manifest_path + self.save() + + def clone_sdk_artifacts_from(self, target): + """ + Clone TargetSDKFile objects from target and associate them with this + target. + """ + sdk_files = target.targetsdkfile_set.all() + for sdk_file in sdk_files: + sdk_file.pk = None + sdk_file.target = self + sdk_file.save() + + def has_images(self): + """ + Returns True if this target has one or more image files attached to it. + """ + return self.target_image_file_set.all().count() > 0 + +# kernel artifacts for a target: bzImage and modules* +class TargetKernelFile(models.Model): + target = models.ForeignKey(Target) + file_name = models.FilePathField() + file_size = models.IntegerField() + + @property + def basename(self): + return os.path.basename(self.file_name) + +# SDK artifacts for a target: sh and manifest files +class TargetSDKFile(models.Model): + target = models.ForeignKey(Target) + file_name = models.FilePathField() + file_size = models.IntegerField() + + @property + def basename(self): + return os.path.basename(self.file_name) + +class Target_Image_File(models.Model): + # valid suffixes for image files produced by a build + SUFFIXES = { + 'btrfs', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma', 'cpio.xz', + 'cramfs', 'elf', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma', 'ext4', + 'ext4.gz', 'ext3', 'ext3.gz', 'hdddirect', 'hddimg', 'iso', 'jffs2', + 'jffs2.sum', 'multiubi', 'qcow2', 'squashfs', 'squashfs-lzo', + 'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi', + 'ubifs', 'vdi', 'vmdk', 'wic', 'wic.bmap', 'wic.bz2', 'wic.gz', 'wic.lzma' + } + + target = models.ForeignKey(Target) + file_name = models.FilePathField(max_length=254) + file_size = models.IntegerField() + + @property + def suffix(self): + """ + Suffix for image file, minus leading "." + """ + for suffix in Target_Image_File.SUFFIXES: + if self.file_name.endswith(suffix): + return suffix + + filename, suffix = os.path.splitext(self.file_name) + suffix = suffix.lstrip('.') + return suffix + +class Target_File(models.Model): + ITYPE_REGULAR = 1 + ITYPE_DIRECTORY = 2 + ITYPE_SYMLINK = 3 + ITYPE_SOCKET = 4 + ITYPE_FIFO = 5 + ITYPE_CHARACTER = 6 + ITYPE_BLOCK = 7 + ITYPES = ( (ITYPE_REGULAR ,'regular'), + ( ITYPE_DIRECTORY ,'directory'), + ( ITYPE_SYMLINK ,'symlink'), + ( ITYPE_SOCKET ,'socket'), + ( ITYPE_FIFO ,'fifo'), + ( ITYPE_CHARACTER ,'character'), + ( ITYPE_BLOCK ,'block'), + ) + + target = models.ForeignKey(Target) + path = models.FilePathField() + size = models.IntegerField() + inodetype = models.IntegerField(choices = ITYPES) + permission = models.CharField(max_length=16) + owner = models.CharField(max_length=128) + group = models.CharField(max_length=128) + directory = models.ForeignKey('Target_File', related_name="directory_set", null=True) + sym_target = models.ForeignKey('Target_File', related_name="symlink_set", null=True) + + +class Task(models.Model): + + SSTATE_NA = 0 + SSTATE_MISS = 1 + SSTATE_FAILED = 2 + SSTATE_RESTORED = 3 + + SSTATE_RESULT = ( + (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking. + (SSTATE_MISS, 'File not in cache'), # the sstate object was not found + (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed + (SSTATE_RESTORED, 'Succeeded'), # successfully restored + ) + + CODING_NA = 0 + CODING_PYTHON = 2 + CODING_SHELL = 3 + + TASK_CODING = ( + (CODING_NA, 'N/A'), + (CODING_PYTHON, 'Python'), + (CODING_SHELL, 'Shell'), + ) + + OUTCOME_NA = -1 + OUTCOME_SUCCESS = 0 + OUTCOME_COVERED = 1 + OUTCOME_CACHED = 2 + OUTCOME_PREBUILT = 3 + OUTCOME_FAILED = 4 + OUTCOME_EMPTY = 5 + + TASK_OUTCOME = ( + (OUTCOME_NA, 'Not Available'), + (OUTCOME_SUCCESS, 'Succeeded'), + (OUTCOME_COVERED, 'Covered'), + (OUTCOME_CACHED, 'Cached'), + (OUTCOME_PREBUILT, 'Prebuilt'), + (OUTCOME_FAILED, 'Failed'), + (OUTCOME_EMPTY, 'Empty'), + ) + + TASK_OUTCOME_HELP = ( + (OUTCOME_SUCCESS, 'This task successfully completed'), + (OUTCOME_COVERED, 'This task did not run because its output is provided by another task'), + (OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'), + (OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'), + (OUTCOME_FAILED, 'This task did not complete'), + (OUTCOME_EMPTY, 'This task has no executable content'), + (OUTCOME_NA, ''), + ) + + search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ] + + def __init__(self, *args, **kwargs): + super(Task, self).__init__(*args, **kwargs) + try: + self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text + except HelpText.DoesNotExist: + self._helptext = None + + def get_related_setscene(self): + return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene") + + def get_outcome_text(self): + return Task.TASK_OUTCOME[int(self.outcome) + 1][1] + + def get_outcome_help(self): + return Task.TASK_OUTCOME_HELP[int(self.outcome)][1] + + def get_sstate_text(self): + if self.sstate_result==Task.SSTATE_NA: + return '' + else: + return Task.SSTATE_RESULT[int(self.sstate_result)][1] + + def get_executed_display(self): + if self.task_executed: + return "Executed" + return "Not Executed" + + def get_description(self): + return self._helptext + + build = models.ForeignKey(Build, related_name='task_build') + order = models.IntegerField(null=True) + task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed + outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA) + sstate_checksum = models.CharField(max_length=100, blank=True) + path_to_sstate_obj = models.FilePathField(max_length=500, blank=True) + recipe = models.ForeignKey('Recipe', related_name='tasks') + task_name = models.CharField(max_length=100) + source_url = models.FilePathField(max_length=255, blank=True) + work_directory = models.FilePathField(max_length=255, blank=True) + script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA) + line_number = models.IntegerField(default=0) + + # start/end times + started = models.DateTimeField(null=True) + ended = models.DateTimeField(null=True) + + # in seconds; this is stored to enable sorting + elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True) + + # in bytes; note that disk_io is stored to enable sorting + disk_io = models.IntegerField(null=True) + disk_io_read = models.IntegerField(null=True) + disk_io_write = models.IntegerField(null=True) + + # in seconds + cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True) + cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True) + + sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA) + message = models.CharField(max_length=240) + logfile = models.FilePathField(max_length=255, blank=True) + + outcome_text = property(get_outcome_text) + sstate_text = property(get_sstate_text) + + def __unicode__(self): + return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name) + + class Meta: + ordering = ('order', 'recipe' ,) + unique_together = ('build', 'recipe', 'task_name', ) + + +class Task_Dependency(models.Model): + task = models.ForeignKey(Task, related_name='task_dependencies_task') + depends_on = models.ForeignKey(Task, related_name='task_dependencies_depends') + +class Package(models.Model): + search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name'] + build = models.ForeignKey('Build', null=True) + recipe = models.ForeignKey('Recipe', null=True) + name = models.CharField(max_length=100) + installed_name = models.CharField(max_length=100, default='') + version = models.CharField(max_length=100, blank=True) + revision = models.CharField(max_length=32, blank=True) + summary = models.TextField(blank=True) + description = models.TextField(blank=True) + size = models.IntegerField(default=0) + installed_size = models.IntegerField(default=0) + section = models.CharField(max_length=80, blank=True) + license = models.CharField(max_length=80, blank=True) + + @property + def is_locale_package(self): + """ Returns True if this package is identifiable as a locale package """ + if self.name.find('locale') != -1: + return True + return False + + @property + def is_packagegroup(self): + """ Returns True is this package is identifiable as a packagegroup """ + if self.name.find('packagegroup') != -1: + return True + return False + +class CustomImagePackage(Package): + # CustomImageRecipe fields to track pacakges appended, + # included and excluded from a CustomImageRecipe + recipe_includes = models.ManyToManyField('CustomImageRecipe', + related_name='includes_set') + recipe_excludes = models.ManyToManyField('CustomImageRecipe', + related_name='excludes_set') + recipe_appends = models.ManyToManyField('CustomImageRecipe', + related_name='appends_set') + + +class Package_DependencyManager(models.Manager): + use_for_related_fields = True + TARGET_LATEST = "use-latest-target-for-target" + + def get_queryset(self): + return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id')) + + def for_target_or_none(self, target): + """ filter the dependencies to be displayed by the supplied target + if no dependences are found for the target then try None as the target + which will return the dependences calculated without the context of a + target e.g. non image recipes. + + returns: { size, packages } + """ + package_dependencies = self.all_depends().order_by('depends_on__name') + + if target is self.TARGET_LATEST: + installed_deps =\ + package_dependencies.filter(~Q(target__target=None)) + else: + installed_deps =\ + package_dependencies.filter(Q(target__target=target)) + + packages_list = None + total_size = 0 + + # If we have installed depdencies for this package and target then use + # these to display + if installed_deps.count() > 0: + packages_list = installed_deps + total_size = installed_deps.aggregate( + Sum('depends_on__size'))['depends_on__size__sum'] + else: + new_list = [] + package_names = [] + + # Find dependencies for the package that we know about even if + # it's not installed on a target e.g. from a non-image recipe + for p in package_dependencies.filter(Q(target=None)): + if p.depends_on.name in package_names: + continue + else: + package_names.append(p.depends_on.name) + new_list.append(p.pk) + # while we're here we may as well total up the size to + # avoid iterating again + total_size += p.depends_on.size + + # We want to return a queryset here for consistency so pick the + # deps from the new_list + packages_list = package_dependencies.filter(Q(pk__in=new_list)) + + return {'packages': packages_list, + 'size': total_size} + + def all_depends(self): + """ Returns just the depends packages and not any other dep_type + Note that this is for any target + """ + return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) | + Q(dep_type=Package_Dependency.TYPE_TRDEPENDS)) + + +class Package_Dependency(models.Model): + TYPE_RDEPENDS = 0 + TYPE_TRDEPENDS = 1 + TYPE_RRECOMMENDS = 2 + TYPE_TRECOMMENDS = 3 + TYPE_RSUGGESTS = 4 + TYPE_RPROVIDES = 5 + TYPE_RREPLACES = 6 + TYPE_RCONFLICTS = 7 + ' TODO: bpackage should be changed to remove the DEPENDS_TYPE access ' + DEPENDS_TYPE = ( + (TYPE_RDEPENDS, "depends"), + (TYPE_TRDEPENDS, "depends"), + (TYPE_TRECOMMENDS, "recommends"), + (TYPE_RRECOMMENDS, "recommends"), + (TYPE_RSUGGESTS, "suggests"), + (TYPE_RPROVIDES, "provides"), + (TYPE_RREPLACES, "replaces"), + (TYPE_RCONFLICTS, "conflicts"), + ) + """ Indexed by dep_type, in view order, key for short name and help + description which when viewed will be printf'd with the + package name. + """ + DEPENDS_DICT = { + TYPE_RDEPENDS : ("depends", "%s is required to run %s"), + TYPE_TRDEPENDS : ("depends", "%s is required to run %s"), + TYPE_TRECOMMENDS : ("recommends", "%s extends the usability of %s"), + TYPE_RRECOMMENDS : ("recommends", "%s extends the usability of %s"), + TYPE_RSUGGESTS : ("suggests", "%s is suggested for installation with %s"), + TYPE_RPROVIDES : ("provides", "%s is provided by %s"), + TYPE_RREPLACES : ("replaces", "%s is replaced by %s"), + TYPE_RCONFLICTS : ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"), + } + + package = models.ForeignKey(Package, related_name='package_dependencies_source') + depends_on = models.ForeignKey(Package, related_name='package_dependencies_target') # soft dependency + dep_type = models.IntegerField(choices=DEPENDS_TYPE) + target = models.ForeignKey(Target, null=True) + objects = Package_DependencyManager() + +class Target_Installed_Package(models.Model): + target = models.ForeignKey(Target) + package = models.ForeignKey(Package, related_name='buildtargetlist_package') + + +class Package_File(models.Model): + package = models.ForeignKey(Package, related_name='buildfilelist_package') + path = models.FilePathField(max_length=255, blank=True) + size = models.IntegerField() + + +class Recipe(models.Model): + search_allowed_fields = ['name', 'version', 'file_path', 'section', + 'summary', 'description', 'license', + 'layer_version__layer__name', + 'layer_version__branch', 'layer_version__commit', + 'layer_version__local_path', + 'layer_version__layer_source'] + + up_date = models.DateTimeField(null=True, default=None) + + name = models.CharField(max_length=100, blank=True) + version = models.CharField(max_length=100, blank=True) + layer_version = models.ForeignKey('Layer_Version', + related_name='recipe_layer_version') + summary = models.TextField(blank=True) + description = models.TextField(blank=True) + section = models.CharField(max_length=100, blank=True) + license = models.CharField(max_length=200, blank=True) + homepage = models.URLField(blank=True) + bugtracker = models.URLField(blank=True) + file_path = models.FilePathField(max_length=255) + pathflags = models.CharField(max_length=200, blank=True) + is_image = models.BooleanField(default=False) + + def __unicode__(self): + return "Recipe " + self.name + ":" + self.version + + def get_vcs_recipe_file_link_url(self): + return self.layer_version.get_vcs_file_link_url(self.file_path) + + def get_description_or_summary(self): + if self.description: + return self.description + elif self.summary: + return self.summary + else: + return "" + + class Meta: + unique_together = (("layer_version", "file_path", "pathflags"), ) + + +class Recipe_DependencyManager(models.Manager): + use_for_related_fields = True + + def get_queryset(self): + return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id')) + +class Provides(models.Model): + name = models.CharField(max_length=100) + recipe = models.ForeignKey(Recipe) + +class Recipe_Dependency(models.Model): + TYPE_DEPENDS = 0 + TYPE_RDEPENDS = 1 + + DEPENDS_TYPE = ( + (TYPE_DEPENDS, "depends"), + (TYPE_RDEPENDS, "rdepends"), + ) + recipe = models.ForeignKey(Recipe, related_name='r_dependencies_recipe') + depends_on = models.ForeignKey(Recipe, related_name='r_dependencies_depends') + via = models.ForeignKey(Provides, null=True, default=None) + dep_type = models.IntegerField(choices=DEPENDS_TYPE) + objects = Recipe_DependencyManager() + + +class Machine(models.Model): + search_allowed_fields = ["name", "description", "layer_version__layer__name"] + up_date = models.DateTimeField(null = True, default = None) + + layer_version = models.ForeignKey('Layer_Version') + name = models.CharField(max_length=255) + description = models.CharField(max_length=255) + + def get_vcs_machine_file_link_url(self): + path = 'conf/machine/'+self.name+'.conf' + + return self.layer_version.get_vcs_file_link_url(path) + + def __unicode__(self): + return "Machine " + self.name + "(" + self.description + ")" + + + + + +class BitbakeVersion(models.Model): + + name = models.CharField(max_length=32, unique = True) + giturl = GitURLField() + branch = models.CharField(max_length=32) + dirpath = models.CharField(max_length=255) + + def __unicode__(self): + return "%s (Branch: %s)" % (self.name, self.branch) + + +class Release(models.Model): + """ A release is a project template, used to pre-populate Project settings with a configuration set """ + name = models.CharField(max_length=32, unique = True) + description = models.CharField(max_length=255) + bitbake_version = models.ForeignKey(BitbakeVersion) + branch_name = models.CharField(max_length=50, default = "") + helptext = models.TextField(null=True) + + def __unicode__(self): + return "%s (%s)" % (self.name, self.branch_name) + + def __str__(self): + return self.name + +class ReleaseDefaultLayer(models.Model): + release = models.ForeignKey(Release) + layer_name = models.CharField(max_length=100, default="") + + +class LayerSource(object): + """ Where the layer metadata came from """ + TYPE_LOCAL = 0 + TYPE_LAYERINDEX = 1 + TYPE_IMPORTED = 2 + TYPE_BUILD = 3 + + SOURCE_TYPE = ( + (TYPE_LOCAL, "local"), + (TYPE_LAYERINDEX, "layerindex"), + (TYPE_IMPORTED, "imported"), + (TYPE_BUILD, "build"), + ) + + def types_dict(): + """ Turn the TYPES enums into a simple dictionary """ + dictionary = {} + for key in LayerSource.__dict__: + if "TYPE" in key: + dictionary[key] = getattr(LayerSource, key) + return dictionary + + +class Layer(models.Model): + + up_date = models.DateTimeField(null=True, default=timezone.now) + + name = models.CharField(max_length=100) + layer_index_url = models.URLField() + vcs_url = GitURLField(default=None, null=True) + local_source_dir = models.TextField(null=True, default=None) + vcs_web_url = models.URLField(null=True, default=None) + vcs_web_tree_base_url = models.URLField(null=True, default=None) + vcs_web_file_base_url = models.URLField(null=True, default=None) + + summary = models.TextField(help_text='One-line description of the layer', + null=True, default=None) + description = models.TextField(null=True, default=None) + + def __unicode__(self): + return "%s / %s " % (self.name, self.summary) + + +class Layer_Version(models.Model): + """ + A Layer_Version either belongs to a single project or no project + """ + search_allowed_fields = ["layer__name", "layer__summary", + "layer__description", "layer__vcs_url", + "dirpath", "release__name", "commit", "branch"] + + build = models.ForeignKey(Build, related_name='layer_version_build', + default=None, null=True) + + layer = models.ForeignKey(Layer, related_name='layer_version_layer') + + layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE, + default=0) + + up_date = models.DateTimeField(null=True, default=timezone.now) + + # To which metadata release does this layer version belong to + release = models.ForeignKey(Release, null=True, default=None) + + branch = models.CharField(max_length=80) + commit = models.CharField(max_length=100) + # If the layer is in a subdir + dirpath = models.CharField(max_length=255, null=True, default=None) + + # if -1, this is a default layer + priority = models.IntegerField(default=0) + + # where this layer exists on the filesystem + local_path = models.FilePathField(max_length=1024, default="/") + + # Set if this layer is restricted to a particular project + project = models.ForeignKey('Project', null=True, default=None) + + # code lifted, with adaptations, from the layerindex-web application + # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/ + def _handle_url_path(self, base_url, path): + import re, posixpath + if base_url: + if self.dirpath: + if path: + extra_path = self.dirpath + '/' + path + # Normalise out ../ in path for usage URL + extra_path = posixpath.normpath(extra_path) + # Minor workaround to handle case where subdirectory has been added between branches + # (should probably support usage URL per branch to handle this... sigh...) + if extra_path.startswith('../'): + extra_path = extra_path[3:] + else: + extra_path = self.dirpath + else: + extra_path = path + branchname = self.release.name + url = base_url.replace('%branch%', branchname) + + # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it + if extra_path: + extra_path = extra_path.replace('%', '%25') + + if '%path%' in base_url: + if extra_path: + url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url) + else: + url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url) + return url.replace('%path%', extra_path) + else: + return url + extra_path + return None + + def get_vcs_link_url(self): + if self.layer.vcs_web_url is None: + return None + return self.layer.vcs_web_url + + def get_vcs_file_link_url(self, file_path=""): + if self.layer.vcs_web_file_base_url is None: + return None + return self._handle_url_path(self.layer.vcs_web_file_base_url, + file_path) + + def get_vcs_dirpath_link_url(self): + if self.layer.vcs_web_tree_base_url is None: + return None + return self._handle_url_path(self.layer.vcs_web_tree_base_url, '') + + def get_vcs_reference(self): + if self.commit is not None and len(self.commit) > 0: + return self.commit + if self.branch is not None and len(self.branch) > 0: + return self.branch + if self.release is not None: + return self.release.name + return 'N/A' + + def get_detailspage_url(self, project_id=None): + """ returns the url to the layer details page uses own project + field if project_id is not specified """ + + if project_id is None: + project_id = self.project.pk + + return reverse('layerdetails', args=(project_id, self.pk)) + + def get_alldeps(self, project_id): + """Get full list of unique layer dependencies.""" + def gen_layerdeps(lver, project, depth): + if depth == 0: + return + for ldep in lver.dependencies.all(): + yield ldep.depends_on + # get next level of deps recursively calling gen_layerdeps + for subdep in gen_layerdeps(ldep.depends_on, project, depth-1): + yield subdep + + project = Project.objects.get(pk=project_id) + result = [] + projectlvers = [player.layercommit for player in + project.projectlayer_set.all()] + # protect against infinite layer dependency loops + maxdepth = 20 + for dep in gen_layerdeps(self, project, maxdepth): + # filter out duplicates and layers already belonging to the project + if dep not in result + projectlvers: + result.append(dep) + + return sorted(result, key=lambda x: x.layer.name) + + def __unicode__(self): + return ("id %d belongs to layer: %s" % (self.pk, self.layer.name)) + + def __str__(self): + if self.release: + release = self.release.name + else: + release = "No release set" + + return "%d %s (%s)" % (self.pk, self.layer.name, release) + + +class LayerVersionDependency(models.Model): + + layer_version = models.ForeignKey(Layer_Version, + related_name="dependencies") + depends_on = models.ForeignKey(Layer_Version, + related_name="dependees") + +class ProjectLayer(models.Model): + project = models.ForeignKey(Project) + layercommit = models.ForeignKey(Layer_Version, null=True) + optional = models.BooleanField(default = True) + + def __unicode__(self): + return "%s, %s" % (self.project.name, self.layercommit) + + class Meta: + unique_together = (("project", "layercommit"),) + +class CustomImageRecipe(Recipe): + + # CustomImageRecipe's belong to layers called: + LAYER_NAME = "toaster-custom-images" + + search_allowed_fields = ['name'] + base_recipe = models.ForeignKey(Recipe, related_name='based_on_recipe') + project = models.ForeignKey(Project) + last_updated = models.DateTimeField(null=True, default=None) + + def get_last_successful_built_target(self): + """ Return the last successful built target object if one exists + otherwise return None """ + return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & + Q(build__project=self.project) & + Q(target=self.name)).last() + + def update_package_list(self): + """ Update the package list from the last good build of this + CustomImageRecipe + """ + # Check if we're aldready up-to-date or not + target = self.get_last_successful_built_target() + if target == None: + # So we've never actually built this Custom recipe but what about + # the recipe it's based on? + target = \ + Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & + Q(build__project=self.project) & + Q(target=self.base_recipe.name)).last() + if target == None: + return + + if target.build.completed_on == self.last_updated: + return + + self.includes_set.clear() + + excludes_list = self.excludes_set.values_list('name', flat=True) + appends_list = self.appends_set.values_list('name', flat=True) + + built_packages_list = \ + target.target_installed_package_set.values_list('package__name', + flat=True) + for built_package in built_packages_list: + # Is the built package in the custom packages list? + if built_package in excludes_list: + continue + + if built_package in appends_list: + continue + + cust_img_p = \ + CustomImagePackage.objects.get(name=built_package) + self.includes_set.add(cust_img_p) + + + self.last_updated = target.build.completed_on + self.save() + + def get_all_packages(self): + """Get the included packages and any appended packages""" + self.update_package_list() + + return CustomImagePackage.objects.filter((Q(recipe_appends=self) | + Q(recipe_includes=self)) & + ~Q(recipe_excludes=self)) + + def get_base_recipe_file(self): + """Get the base recipe file path if it exists on the file system""" + path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path, + self.base_recipe.file_path) + + path_schema_two = self.base_recipe.file_path + + if os.path.exists(path_schema_one): + return path_schema_one + + # The path may now be the full path if the recipe has been built + if os.path.exists(path_schema_two): + return path_schema_two + + return None + + def generate_recipe_file_contents(self): + """Generate the contents for the recipe file.""" + # If we have no excluded packages we only need to _append + if self.excludes_set.count() == 0: + packages_conf = "IMAGE_INSTALL_append = \" " + + for pkg in self.appends_set.all(): + packages_conf += pkg.name+' ' + else: + packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \"" + # We add all the known packages to be built by this recipe apart + # from locale packages which are are controlled with IMAGE_LINGUAS. + for pkg in self.get_all_packages().exclude( + name__icontains="locale"): + packages_conf += pkg.name+' ' + + packages_conf += "\"" + + base_recipe_path = self.get_base_recipe_file() + if base_recipe_path: + base_recipe = open(base_recipe_path, 'r').read() + else: + raise IOError("Based on recipe file not found: %s" % + base_recipe_path) + + # Add a special case for when the recipe we have based a custom image + # recipe on requires another recipe. + # For example: + # "require core-image-minimal.bb" is changed to: + # "require recipes-core/images/core-image-minimal.bb" + + req_search = re.search(r'(require\s+)(.+\.bb\s*$)', + base_recipe, + re.MULTILINE) + if req_search: + require_filename = req_search.group(2).strip() + + corrected_location = Recipe.objects.filter( + Q(layer_version=self.base_recipe.layer_version) & + Q(file_path__icontains=require_filename)).last().file_path + + new_require_line = "require %s" % corrected_location + + base_recipe = base_recipe.replace(req_search.group(0), + new_require_line) + + info = { + "date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"), + "base_recipe": base_recipe, + "recipe_name": self.name, + "base_recipe_name": self.base_recipe.name, + "license": self.license, + "summary": self.summary, + "description": self.description, + "packages_conf": packages_conf.strip() + } + + recipe_contents = ("# Original recipe %(base_recipe_name)s \n" + "%(base_recipe)s\n\n" + "# Recipe %(recipe_name)s \n" + "# Customisation Generated by Toaster on %(date)s\n" + "SUMMARY = \"%(summary)s\"\n" + "DESCRIPTION = \"%(description)s\"\n" + "LICENSE = \"%(license)s\"\n" + "%(packages_conf)s") % info + + return recipe_contents + +class ProjectVariable(models.Model): + project = models.ForeignKey(Project) + name = models.CharField(max_length=100) + value = models.TextField(blank = True) + +class Variable(models.Model): + search_allowed_fields = ['variable_name', 'variable_value', + 'vhistory__file_name', "description"] + build = models.ForeignKey(Build, related_name='variable_build') + variable_name = models.CharField(max_length=100) + variable_value = models.TextField(blank=True) + changed = models.BooleanField(default=False) + human_readable_name = models.CharField(max_length=200) + description = models.TextField(blank=True) + +class VariableHistory(models.Model): + variable = models.ForeignKey(Variable, related_name='vhistory') + value = models.TextField(blank=True) + file_name = models.FilePathField(max_length=255) + line_number = models.IntegerField(null=True) + operation = models.CharField(max_length=64) + +class HelpText(models.Model): + VARIABLE = 0 + HELPTEXT_AREA = ((VARIABLE, 'variable'), ) + + build = models.ForeignKey(Build, related_name='helptext_build') + area = models.IntegerField(choices=HELPTEXT_AREA) + key = models.CharField(max_length=100) + text = models.TextField() + +class LogMessage(models.Model): + EXCEPTION = -1 # used to signal self-toaster-exceptions + INFO = 0 + WARNING = 1 + ERROR = 2 + CRITICAL = 3 + + LOG_LEVEL = ( + (INFO, "info"), + (WARNING, "warn"), + (ERROR, "error"), + (CRITICAL, "critical"), + (EXCEPTION, "toaster exception") + ) + + build = models.ForeignKey(Build) + task = models.ForeignKey(Task, blank = True, null=True) + level = models.IntegerField(choices=LOG_LEVEL, default=INFO) + message = models.TextField(blank=True, null=True) + pathname = models.FilePathField(max_length=255, blank=True) + lineno = models.IntegerField(null=True) + + def __str__(self): + return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build)) + +def invalidate_cache(**kwargs): + from django.core.cache import cache + try: + cache.clear() + except Exception as e: + logger.warning("Problem with cache backend: Failed to clear cache: %s" % e) + +def signal_runbuilds(): + """Send SIGUSR1 to runbuilds process""" + try: + with open(os.path.join(os.getenv('BUILDDIR', '.'), + '.runbuilds.pid')) as pidf: + os.kill(int(pidf.read()), SIGUSR1) + except FileNotFoundError: + logger.info("Stopping existing runbuilds: no current process found") + +class Distro(models.Model): + search_allowed_fields = ["name", "description", "layer_version__layer__name"] + up_date = models.DateTimeField(null = True, default = None) + + layer_version = models.ForeignKey('Layer_Version') + name = models.CharField(max_length=255) + description = models.CharField(max_length=255) + + def get_vcs_distro_file_link_url(self): + path = self.name+'.conf' + return self.layer_version.get_vcs_file_link_url(path) + + def __unicode__(self): + return "Distro " + self.name + "(" + self.description + ")" + +django.db.models.signals.post_save.connect(invalidate_cache) +django.db.models.signals.post_delete.connect(invalidate_cache) +django.db.models.signals.m2m_changed.connect(invalidate_cache) diff --git a/poky/bitbake/lib/toaster/tests/__init__.py b/poky/bitbake/lib/toaster/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/tests/browser/README b/poky/bitbake/lib/toaster/tests/browser/README new file mode 100644 index 000000000..352c4fe3e --- /dev/null +++ b/poky/bitbake/lib/toaster/tests/browser/README @@ -0,0 +1,74 @@ +# Running Toaster's browser-based test suite + +These tests require Selenium to be installed in your Python environment. + +The simplest way to install this is via pip3: + + pip3 install selenium==2.53.2 + +Note that if you use other versions of Selenium, some of the tests (such as +tests.browser.test_js_unit_tests.TestJsUnitTests) may fail, as these rely on +a Selenium test report with a version-specific format. + +To run tests against Chrome: + +* Download chromedriver for your host OS from + https://sites.google.com/a/chromium.org/chromedriver/downloads +* On *nix systems, put chromedriver on PATH +* On Windows, put chromedriver.exe in the same directory as chrome.exe + +To run tests against PhantomJS (headless): +--NOTE - Selenium seems to be deprecating support for this mode --- +* Download and install PhantomJS: + http://phantomjs.org/download.html +* On *nix systems, put phantomjs on PATH +* Not tested on Windows + +To run tests against Firefox, you may need to install the Marionette driver, +depending on how new your version of Firefox is. One clue that you need to do +this is if you see an exception like: + + selenium.common.exceptions.WebDriverException: Message: The browser + appears to have exited before we could connect. If you specified + a log_file in the FirefoxBinary constructor, check it for details. + +See https://developer.mozilla.org/en-US/docs/Mozilla/QA/Marionette/WebDriver +for installation instructions. Ensure that the Marionette executable (renamed +as wires on Linux or wires.exe on Windows) is on your PATH; and use "marionette" +as the browser string passed via TOASTER_TESTS_BROWSER (see below). + +(Note: The Toaster tests have been checked against Firefox 47 with the +Marionette driver.) + +The test cases will instantiate a Selenium driver set by the +TOASTER_TESTS_BROWSER environment variable, or Chrome if this is not specified. + +To run tests against the Selenium Firefox Docker container: +More explanation is located at https://wiki.yoctoproject.org/wiki/TipsAndTricks/TestingToasterWithContainers +* Run the Selenium container: + ** docker run -it --rm=true -p 5900:5900 -p 4444:4444 --name=selenium selenium/standalone-firefox-debug:2.53.0 + *** 5900 is the default vnc port. If you are runing a vnc server on your machine map a different port e.g. -p 6900:5900 and connect vnc client to 127.0.0.1:6900 + *** 4444 is the default selenium sever port. +* Run the tests + ** TOASTER_TESTS_BROWSER=http://127.0.0.1:4444/wd/hub TOASTER_TESTS_URL=http://172.17.0.1:8000 ./bitbake/lib/toaster/manage.py test --liveserver=172.17.0.1:8000 tests.browser + ** TOASTER_TESTS_BROWSER=remote TOASTER_REMOTE_HUB=http://127.0.0.1:4444/wd/hub ./bitbake/lib/toaster/manage.py test --liveserver=172.17.0.1:8000 tests.browser + *** TOASTER_REMOTE_HUB - This is the address for the Selenium Remote Web Driver hub. Assuming you ran the contianer with -p 4444:4444 it will be http://127.0.0.1:4444/wd/hub. + *** --liveserver=xxx tells Django to run the test server on an interface and port reachable by both host and container. + **** 172.17.0.1 is the default docker bridge on linux, viewable from inside and outside the contianers. Find it with "ip -4 addr show dev docker0" +* connect to the vnc server to see the tests if you would like + ** xtightvncviewer 127.0.0.1:5900 + ** note, you need to wait for the test container to come up before this can connect. + +Available drivers: + +* chrome (default) +* firefox +* marionette (for newer Firefoxes) +* ie +* phantomjs (deprecated) +* remote + +e.g. to run the test suite with phantomjs where you have phantomjs installed +in /home/me/apps/phantomjs: + +PATH=/home/me/apps/phantomjs/bin:$PATH TOASTER_TESTS_BROWSER=phantomjs manage.py test tests.browser diff --git a/poky/bitbake/lib/toaster/tests/browser/__init__.py b/poky/bitbake/lib/toaster/tests/browser/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poky/bitbake/lib/toaster/tests/browser/selenium_helpers.py b/poky/bitbake/lib/toaster/tests/browser/selenium_helpers.py new file mode 100644 index 000000000..08711e455 --- /dev/null +++ b/poky/bitbake/lib/toaster/tests/browser/selenium_helpers.py @@ -0,0 +1,34 @@ +#! /usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# The Wait class and some of SeleniumDriverHelper and SeleniumTestCase are +# modified from Patchwork, released under the same licence terms as Toaster: +# https://github.com/dlespiau/patchwork/blob/master/patchwork/tests.browser.py + +""" +Helper methods for creating Toaster Selenium tests which run within +the context of Django unit tests. +""" +from django.contrib.staticfiles.testing import StaticLiveServerTestCase +from tests.browser.selenium_helpers_base import SeleniumTestCaseBase + +class SeleniumTestCase(SeleniumTestCaseBase, StaticLiveServerTestCase): + pass diff --git a/poky/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py b/poky/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py new file mode 100644 index 000000000..156d639b1 --- /dev/null +++ b/poky/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py @@ -0,0 +1,227 @@ +#! /usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# The Wait class and some of SeleniumDriverHelper and SeleniumTestCase are +# modified from Patchwork, released under the same licence terms as Toaster: +# https://github.com/dlespiau/patchwork/blob/master/patchwork/tests.browser.py + +""" +Helper methods for creating Toaster Selenium tests which run within +the context of Django unit tests. +""" + +import os +import time +import unittest + +from django.contrib.staticfiles.testing import StaticLiveServerTestCase +from selenium import webdriver +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.common.exceptions import NoSuchElementException, \ + StaleElementReferenceException, TimeoutException + +def create_selenium_driver(cls,browser='chrome'): + # set default browser string based on env (if available) + env_browser = os.environ.get('TOASTER_TESTS_BROWSER') + if env_browser: + browser = env_browser + + if browser == 'chrome': + return webdriver.Chrome( + service_args=["--verbose", "--log-path=selenium.log"] + ) + elif browser == 'firefox': + return webdriver.Firefox() + elif browser == 'marionette': + capabilities = DesiredCapabilities.FIREFOX + capabilities['marionette'] = True + return webdriver.Firefox(capabilities=capabilities) + elif browser == 'ie': + return webdriver.Ie() + elif browser == 'phantomjs': + return webdriver.PhantomJS() + elif browser == 'remote': + # if we were to add yet another env variable like TOASTER_REMOTE_BROWSER + # we could let people pick firefox or chrome, left for later + remote_hub= os.environ.get('TOASTER_REMOTE_HUB') + driver = webdriver.Remote(remote_hub, + webdriver.DesiredCapabilities.FIREFOX.copy()) + + driver.get("http://%s:%s"%(cls.server_thread.host,cls.server_thread.port)) + return driver + else: + msg = 'Selenium driver for browser %s is not available' % browser + raise RuntimeError(msg) + +class Wait(WebDriverWait): + """ + Subclass of WebDriverWait with predetermined timeout and poll + frequency. Also deals with a wider variety of exceptions. + """ + _TIMEOUT = 10 + _POLL_FREQUENCY = 0.5 + + def __init__(self, driver): + super(Wait, self).__init__(driver, self._TIMEOUT, self._POLL_FREQUENCY) + + def until(self, method, message=''): + """ + Calls the method provided with the driver as an argument until the + return value is not False. + """ + + end_time = time.time() + self._timeout + while True: + try: + value = method(self._driver) + if value: + return value + except NoSuchElementException: + pass + except StaleElementReferenceException: + pass + + time.sleep(self._poll) + if time.time() > end_time: + break + + raise TimeoutException(message) + + def until_not(self, method, message=''): + """ + Calls the method provided with the driver as an argument until the + return value is False. + """ + + end_time = time.time() + self._timeout + while True: + try: + value = method(self._driver) + if not value: + return value + except NoSuchElementException: + return True + except StaleElementReferenceException: + pass + + time.sleep(self._poll) + if time.time() > end_time: + break + + raise TimeoutException(message) + +class SeleniumTestCaseBase(unittest.TestCase): + """ + NB StaticLiveServerTestCase is used as the base test case so that + static files are served correctly in a Selenium test run context; see + https://docs.djangoproject.com/en/1.9/ref/contrib/staticfiles/#specialized-test-case-to-support-live-testing + """ + + @classmethod + def setUpClass(cls): + """ Create a webdriver driver at the class level """ + + super(SeleniumTestCaseBase, cls).setUpClass() + + # instantiate the Selenium webdriver once for all the test methods + # in this test case + cls.driver = create_selenium_driver(cls) + cls.driver.maximize_window() + + @classmethod + def tearDownClass(cls): + """ Clean up webdriver driver """ + + cls.driver.quit() + super(SeleniumTestCaseBase, cls).tearDownClass() + + def get(self, url): + """ + Selenium requires absolute URLs, so convert Django URLs returned + by resolve() or similar to absolute ones and get using the + webdriver instance. + + url: a relative URL + """ + abs_url = '%s%s' % (self.live_server_url, url) + self.driver.get(abs_url) + + def find(self, selector): + """ Find single element by CSS selector """ + return self.driver.find_element_by_css_selector(selector) + + def find_all(self, selector): + """ Find all elements matching CSS selector """ + return self.driver.find_elements_by_css_selector(selector) + + def element_exists(self, selector): + """ + Return True if one element matching selector exists, + False otherwise + """ + return len(self.find_all(selector)) == 1 + + def focused_element(self): + """ Return the element which currently has focus on the page """ + return self.driver.switch_to.active_element + + def wait_until_present(self, selector): + """ Wait until element matching CSS selector is on the page """ + is_present = lambda driver: self.find(selector) + msg = 'An element matching "%s" should be on the page' % selector + element = Wait(self.driver).until(is_present, msg) + return element + + def wait_until_visible(self, selector): + """ Wait until element matching CSS selector is visible on the page """ + is_visible = lambda driver: self.find(selector).is_displayed() + msg = 'An element matching "%s" should be visible' % selector + Wait(self.driver).until(is_visible, msg) + return self.find(selector) + + def wait_until_focused(self, selector): + """ Wait until element matching CSS selector has focus """ + is_focused = \ + lambda driver: self.find(selector) == self.focused_element() + msg = 'An element matching "%s" should be focused' % selector + Wait(self.driver).until(is_focused, msg) + return self.find(selector) + + def enter_text(self, selector, value): + """ Insert text into element matching selector """ + # note that keyup events don't occur until the element is clicked + # (in the case of , for example), so simulate + # user clicking the element before inserting text into it + field = self.click(selector) + + field.send_keys(value) + return field + + def click(self, selector): + """ Click on element which matches CSS selector """ + element = self.wait_until_visible(selector) + element.click() + return element + + def get_page_source(self): + """ Get raw HTML for the current page """ + return self.driver.page_source diff --git a/poky/bitbake/lib/toaster/tests/browser/test_all_builds_page.py b/poky/bitbake/lib/toaster/tests/browser/test_all_builds_page.py new file mode 100644 index 000000000..b86f29bdd --- /dev/null +++ b/poky/bitbake/lib/toaster/tests/browser/test_all_builds_page.py @@ -0,0 +1,233 @@ +#! /usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import re + +from django.core.urlresolvers import reverse +from django.utils import timezone +from tests.browser.selenium_helpers import SeleniumTestCase + +from orm.models import BitbakeVersion, Release, Project, Build, Target + + +class TestAllBuildsPage(SeleniumTestCase): + """ Tests for all builds page /builds/ """ + + PROJECT_NAME = 'test project' + CLI_BUILDS_PROJECT_NAME = 'command line builds' + + def setUp(self): + bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/', + branch='master', dirpath='') + release = Release.objects.create(name='release1', + bitbake_version=bbv) + self.project1 = Project.objects.create_project(name=self.PROJECT_NAME, + release=release) + self.default_project = Project.objects.create_project( + name=self.CLI_BUILDS_PROJECT_NAME, + release=release + ) + self.default_project.is_default = True + self.default_project.save() + + # parameters for builds to associate with the projects + now = timezone.now() + + self.project1_build_success = { + 'project': self.project1, + 'started_on': now, + 'completed_on': now, + 'outcome': Build.SUCCEEDED + } + + self.project1_build_failure = { + 'project': self.project1, + 'started_on': now, + 'completed_on': now, + 'outcome': Build.FAILED + } + + self.default_project_build_success = { + 'project': self.default_project, + 'started_on': now, + 'completed_on': now, + 'outcome': Build.SUCCEEDED + } + + def _get_build_time_element(self, build): + """ + Return the HTML element containing the build time for a build + in the recent builds area + """ + selector = 'div[data-latest-build-result="%s"] ' \ + '[data-role="data-recent-build-buildtime-field"]' % build.id + + # because this loads via Ajax, wait for it to be visible + self.wait_until_present(selector) + + build_time_spans = self.find_all(selector) + + self.assertEqual(len(build_time_spans), 1) + + return build_time_spans[0] + + def _get_row_for_build(self, build): + """ Get the table row for the build from the all builds table """ + self.wait_until_present('#allbuildstable') + + rows = self.find_all('#allbuildstable tr') + + # look for the row with a download link on the recipe which matches the + # build ID + url = reverse('builddashboard', args=(build.id,)) + selector = 'td.target a[href="%s"]' % url + + found_row = None + for row in rows: + + outcome_links = row.find_elements_by_css_selector(selector) + if len(outcome_links) == 1: + found_row = row + break + + self.assertNotEqual(found_row, None) + + return found_row + + def test_show_tasks_with_suffix(self): + """ Task should be shown as suffix on build name """ + build = Build.objects.create(**self.project1_build_success) + target = 'bash' + task = 'clean' + Target.objects.create(build=build, target=target, task=task) + + url = reverse('all-builds') + self.get(url) + self.wait_until_present('td[class="target"]') + + cell = self.find('td[class="target"]') + content = cell.get_attribute('innerHTML') + expected_text = '%s:%s' % (target, task) + + self.assertTrue(re.search(expected_text, content), + '"target" cell should contain text %s' % expected_text) + + def test_rebuild_buttons(self): + """ + Test 'Rebuild' buttons in recent builds section + + 'Rebuild' button should not be shown for command-line builds, + but should be shown for other builds + """ + build1 = Build.objects.create(**self.project1_build_success) + default_build = Build.objects.create(**self.default_project_build_success) + + url = reverse('all-builds') + self.get(url) + + # shouldn't see a rebuild button for command-line builds + selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % default_build.id + run_again_button = self.find_all(selector) + self.assertEqual(len(run_again_button), 0, + 'should not see a rebuild button for cli builds') + + # should see a rebuild button for non-command-line builds + selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % build1.id + run_again_button = self.find_all(selector) + self.assertEqual(len(run_again_button), 1, + 'should see a rebuild button for non-cli builds') + + def test_tooltips_on_project_name(self): + """ + Test tooltips shown next to project name in the main table + + A tooltip should be present next to the command line + builds project name in the all builds page, but not for + other projects + """ + Build.objects.create(**self.project1_build_success) + Build.objects.create(**self.default_project_build_success) + + url = reverse('all-builds') + self.get(url) + + # get the project name cells from the table + cells = self.find_all('#allbuildstable td[class="project"]') + + selector = 'span.get-help' + + for cell in cells: + content = cell.get_attribute('innerHTML') + help_icons = cell.find_elements_by_css_selector(selector) + + if re.search(self.PROJECT_NAME, content): + # no help icon next to non-cli project name + msg = 'should not be a help icon for non-cli builds name' + self.assertEqual(len(help_icons), 0, msg) + elif re.search(self.CLI_BUILDS_PROJECT_NAME, content): + # help icon next to cli project name + msg = 'should be a help icon for cli builds name' + self.assertEqual(len(help_icons), 1, msg) + else: + msg = 'found unexpected project name cell in all builds table' + self.fail(msg) + + def test_builds_time_links(self): + """ + Successful builds should have links on the time column and in the + recent builds area; failed builds should not have links on the time column, + or in the recent builds area + """ + build1 = Build.objects.create(**self.project1_build_success) + build2 = Build.objects.create(**self.project1_build_failure) + + # add some targets to these builds so they have recipe links + # (and so we can find the row in the ToasterTable corresponding to + # a particular build) + Target.objects.create(build=build1, target='foo') + Target.objects.create(build=build2, target='bar') + + url = reverse('all-builds') + self.get(url) + + # test recent builds area for successful build + element = self._get_build_time_element(build1) + links = element.find_elements_by_css_selector('a') + msg = 'should be a link on the build time for a successful recent build' + self.assertEquals(len(links), 1, msg) + + # test recent builds area for failed build + element = self._get_build_time_element(build2) + links = element.find_elements_by_css_selector('a') + msg = 'should not be a link on the build time for a failed recent build' + self.assertEquals(len(links), 0, msg) + + # test the time column for successful build + build1_row = self._get_row_for_build(build1) + links = build1_row.find_elements_by_css_selector('td.time a') + msg = 'should be a link on the build time for a successful build' + self.assertEquals(len(links), 1, msg) + + # test the time column for failed build + build2_row = self._get_row_for_build(build2) + links = build2_row.find_elements_by_css_selector('td.time a') + msg = 'should not be a link on the build time for a failed build' + self.assertEquals(len(links), 0, msg) diff --git a/poky/bitbake/lib/toaster/tests/browser/test_all_projects_page.py b/poky/bitbake/lib/toaster/tests/browser/test_all_projects_page.py new file mode 100644 index 000000000..44da64075 --- /dev/null +++ b/poky/bitbake/lib/toaster/tests/browser/test_all_projects_page.py @@ -0,0 +1,217 @@ +#! /usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import re + +from django.core.urlresolvers import reverse +from django.utils import timezone +from tests.browser.selenium_helpers import SeleniumTestCase + +from orm.models import BitbakeVersion, Release, Project, Build +from orm.models import ProjectVariable + +class TestAllProjectsPage(SeleniumTestCase): + """ Browser tests for projects page /projects/ """ + + PROJECT_NAME = 'test project' + CLI_BUILDS_PROJECT_NAME = 'command line builds' + MACHINE_NAME = 'delorean' + + def setUp(self): + """ Add default project manually """ + project = Project.objects.create_project(self.CLI_BUILDS_PROJECT_NAME, None) + self.default_project = project + self.default_project.is_default = True + self.default_project.save() + + # this project is only set for some of the tests + self.project = None + + self.release = None + + def _add_build_to_default_project(self): + """ Add a build to the default project (not used in all tests) """ + now = timezone.now() + build = Build.objects.create(project=self.default_project, + started_on=now, + completed_on=now) + build.save() + + def _add_non_default_project(self): + """ Add another project """ + bbv = BitbakeVersion.objects.create(name='test bbv', giturl='/tmp/', + branch='master', dirpath='') + self.release = Release.objects.create(name='test release', + branch_name='master', + bitbake_version=bbv) + self.project = Project.objects.create_project(self.PROJECT_NAME, self.release) + self.project.is_default = False + self.project.save() + + # fake the MACHINE variable + project_var = ProjectVariable.objects.create(project=self.project, + name='MACHINE', + value=self.MACHINE_NAME) + project_var.save() + + def _get_row_for_project(self, project_name): + """ Get the HTML row for a project, or None if not found """ + self.wait_until_present('#projectstable tbody tr') + rows = self.find_all('#projectstable tbody tr') + + # find the row with a project name matching the one supplied + found_row = None + for row in rows: + if re.search(project_name, row.get_attribute('innerHTML')): + found_row = row + break + + return found_row + + def test_default_project_hidden(self): + """ + The default project should be hidden if it has no builds + and we should see the "no results" area + """ + url = reverse('all-projects') + self.get(url) + self.wait_until_visible('#empty-state-projectstable') + + rows = self.find_all('#projectstable tbody tr') + self.assertEqual(len(rows), 0, 'should be no projects displayed') + + def test_default_project_has_build(self): + """ The default project should be shown if it has builds """ + self._add_build_to_default_project() + + url = reverse('all-projects') + self.get(url) + + default_project_row = self._get_row_for_project(self.default_project.name) + + self.assertNotEqual(default_project_row, None, + 'default project "cli builds" should be in page') + + def test_default_project_release(self): + """ + The release for the default project should display as + 'Not applicable' + """ + # need a build, otherwise project doesn't display at all + self._add_build_to_default_project() + + # another project to test, which should show release + self._add_non_default_project() + + self.get(reverse('all-projects')) + self.wait_until_visible("#projectstable tr") + + # find the row for the default project + default_project_row = self._get_row_for_project(self.default_project.name) + + # check the release text for the default project + selector = 'span[data-project-field="release"] span.text-muted' + element = default_project_row.find_element_by_css_selector(selector) + text = element.text.strip() + self.assertEqual(text, 'Not applicable', + 'release should be "not applicable" for default project') + + # find the row for the default project + other_project_row = self._get_row_for_project(self.project.name) + + # check the link in the release cell for the other project + selector = 'span[data-project-field="release"]' + element = other_project_row.find_element_by_css_selector(selector) + text = element.text.strip() + self.assertEqual(text, self.release.name, + 'release name should be shown for non-default project') + + def test_default_project_machine(self): + """ + The machine for the default project should display as + 'Not applicable' + """ + # need a build, otherwise project doesn't display at all + self._add_build_to_default_project() + + # another project to test, which should show machine + self._add_non_default_project() + + self.get(reverse('all-projects')) + + self.wait_until_visible("#projectstable tr") + + # find the row for the default project + default_project_row = self._get_row_for_project(self.default_project.name) + + # check the machine cell for the default project + selector = 'span[data-project-field="machine"] span.text-muted' + element = default_project_row.find_element_by_css_selector(selector) + text = element.text.strip() + self.assertEqual(text, 'Not applicable', + 'machine should be not applicable for default project') + + # find the row for the default project + other_project_row = self._get_row_for_project(self.project.name) + + # check the link in the machine cell for the other project + selector = 'span[data-project-field="machine"]' + element = other_project_row.find_element_by_css_selector(selector) + text = element.text.strip() + self.assertEqual(text, self.MACHINE_NAME, + 'machine name should be shown for non-default project') + + def test_project_page_links(self): + """ + Test that links for the default project point to the builds + page /projects/X/builds for that project, and that links for + other projects point to their configuration pages /projects/X/ + """ + + # need a build, otherwise project doesn't display at all + self._add_build_to_default_project() + + # another project to test + self._add_non_default_project() + + self.get(reverse('all-projects')) + + # find the row for the default project + default_project_row = self._get_row_for_project(self.default_project.name) + + # check the link on the name field + selector = 'span[data-project-field="name"] a' + element = default_project_row.find_element_by_css_selector(selector) + link_url = element.get_attribute('href').strip() + expected_url = reverse('projectbuilds', args=(self.default_project.id,)) + msg = 'link on default project name should point to builds but was %s' % link_url + self.assertTrue(link_url.endswith(expected_url), msg) + + # find the row for the other project + other_project_row = self._get_row_for_project(self.project.name) + + # check the link for the other project + selector = 'span[data-project-field="name"] a' + element = other_project_row.find_element_by_css_selector(selector) + link_url = element.get_attribute('href').strip() + expected_url = reverse('project', args=(self.project.id,)) + msg = 'link on project name should point to configuration but was %s' % link_url + self.assertTrue(link_url.endswith(expected_url), msg) diff --git a/poky/bitbake/lib/toaster/tests/browser/test_builddashboard_page.py b/poky/bitbake/lib/toaster/tests/browser/test_builddashboard_page.py new file mode 100644 index 000000000..f8ccb5452 --- /dev/null +++ b/poky/bitbake/lib/toaster/tests/browser/test_builddashboard_page.py @@ -0,0 +1,347 @@ +#! /usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from django.core.urlresolvers import reverse +from django.utils import timezone + +from tests.browser.selenium_helpers import SeleniumTestCase + +from orm.models import Project, Release, BitbakeVersion, Build, LogMessage +from orm.models import Layer, Layer_Version, Recipe, CustomImageRecipe, Variable + +class TestBuildDashboardPage(SeleniumTestCase): + """ Tests for the build dashboard /build/X """ + + def setUp(self): + bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/', + branch='master', dirpath="") + release = Release.objects.create(name='release1', + bitbake_version=bbv) + project = Project.objects.create_project(name='test project', + release=release) + + now = timezone.now() + + self.build1 = Build.objects.create(project=project, + started_on=now, + completed_on=now, + outcome=Build.SUCCEEDED) + + self.build2 = Build.objects.create(project=project, + started_on=now, + completed_on=now, + outcome=Build.SUCCEEDED) + + self.build3 = Build.objects.create(project=project, + started_on=now, + completed_on=now, + outcome=Build.FAILED) + + # add Variable objects to the successful builds, as this is the criterion + # used to determine whether the left-hand panel should be displayed + Variable.objects.create(build=self.build1, + variable_name='Foo', + variable_value='Bar') + Variable.objects.create(build=self.build2, + variable_name='Foo', + variable_value='Bar') + + # exception + msg1 = 'an exception was thrown' + self.exception_message = LogMessage.objects.create( + build=self.build1, + level=LogMessage.EXCEPTION, + message=msg1 + ) + + # critical + msg2 = 'a critical error occurred' + self.critical_message = LogMessage.objects.create( + build=self.build1, + level=LogMessage.CRITICAL, + message=msg2 + ) + + # error on the failed build + msg3 = 'an error occurred' + self.error_message = LogMessage.objects.create( + build=self.build3, + level=LogMessage.ERROR, + message=msg3 + ) + + # warning on the failed build + msg4 = 'DANGER WILL ROBINSON' + self.warning_message = LogMessage.objects.create( + build=self.build3, + level=LogMessage.WARNING, + message=msg4 + ) + + # recipes related to the build, for testing the edit custom image/new + # custom image buttons + layer = Layer.objects.create(name='alayer') + layer_version = Layer_Version.objects.create( + layer=layer, build=self.build1 + ) + + # non-image recipes related to a build, for testing the new custom + # image button + layer_version2 = Layer_Version.objects.create(layer=layer, + build=self.build3) + + # image recipes + self.image_recipe1 = Recipe.objects.create( + name='recipeA', + layer_version=layer_version, + file_path='/foo/recipeA.bb', + is_image=True + ) + self.image_recipe2 = Recipe.objects.create( + name='recipeB', + layer_version=layer_version, + file_path='/foo/recipeB.bb', + is_image=True + ) + + # custom image recipes for this project + self.custom_image_recipe1 = CustomImageRecipe.objects.create( + name='customRecipeY', + project=project, + layer_version=layer_version, + file_path='/foo/customRecipeY.bb', + base_recipe=self.image_recipe1, + is_image=True + ) + self.custom_image_recipe2 = CustomImageRecipe.objects.create( + name='customRecipeZ', + project=project, + layer_version=layer_version, + file_path='/foo/customRecipeZ.bb', + base_recipe=self.image_recipe2, + is_image=True + ) + + # custom image recipe for a different project (to test filtering + # of image recipes and custom image recipes is correct: this shouldn't + # show up in either query against self.build1) + self.custom_image_recipe3 = CustomImageRecipe.objects.create( + name='customRecipeOmega', + project=Project.objects.create(name='baz', release=release), + layer_version=Layer_Version.objects.create( + layer=layer, build=self.build2 + ), + file_path='/foo/customRecipeOmega.bb', + base_recipe=self.image_recipe2, + is_image=True + ) + + # another non-image recipe (to test filtering of image recipes and + # custom image recipes is correct: this shouldn't show up in either + # for any build) + self.non_image_recipe = Recipe.objects.create( + name='nonImageRecipe', + layer_version=layer_version, + file_path='/foo/nonImageRecipe.bb', + is_image=False + ) + + def _get_build_dashboard(self, build): + """ + Navigate to the build dashboard for build + """ + url = reverse('builddashboard', args=(build.id,)) + self.get(url) + + def _get_build_dashboard_errors(self, build): + """ + Get a list of HTML fragments representing the errors on the + dashboard for the Build object build + """ + self._get_build_dashboard(build) + return self.find_all('#errors div.alert-danger') + + def _check_for_log_message(self, message_elements, log_message): + """ + Check that the LogMessage has a representation in + the HTML elements . + + message_elements: WebElements representing the log messages shown + in the build dashboard; each should have a 
 element inside
+        it with a data-log-message-id attribute
+
+        log_message: orm.models.LogMessage instance
+        """
+        expected_text = log_message.message
+        expected_pk = str(log_message.pk)
+
+        found = False
+        for element in message_elements:
+            log_message_text = element.find_element_by_tag_name('pre').text.strip()
+            text_matches = (log_message_text == expected_text)
+
+            log_message_pk = element.get_attribute('data-log-message-id')
+            id_matches = (log_message_pk == expected_pk)
+
+            if text_matches and id_matches:
+                found = True
+                break
+
+        template_vars = (expected_text, expected_pk)
+        assertion_failed_msg = 'message not found: ' \
+            'expected text "%s" and ID %s' % template_vars
+        self.assertTrue(found, assertion_failed_msg)
+
+    def _check_for_error_message(self, build, log_message):
+        """
+        Check whether the LogMessage instance  is
+        represented as an HTML error in the dashboard page for the Build object
+        build
+        """
+        errors = self._get_build_dashboard_errors(build)
+        self._check_for_log_message(errors, log_message)
+
+    def _check_labels_in_modal(self, modal, expected):
+        """
+        Check that the text values of the