[{"_1":2,"_175":-5,"_176":-5},"loaderData",{"_3":4,"_13":14,"_21":22,"_154":155},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":17,"_18":17,"_19":11,"_20":11},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_23":24,"_25":-5,"_26":11,"_27":11,"_28":29,"_30":31,"_32":-5,"_33":11,"_34":35,"_84":85,"_107":108,"_145":-5,"_146":11,"_147":17,"_148":11,"_149":-5,"_150":11,"_151":-5,"_152":153},"canonicalUrl","https://hub.docker.com/r/windriver/toaster_aggregator/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","windriver","reponame","toaster_aggregator","cloudBuildSource","isAutobuildsAvailable","repo",{"_36":37,"_38":39,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":51,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":70,"_75":17,"_76":77,"_81":29,"_82":83},"categories",[],"content_types",[40],"image","date_registered","2017-09-29T19:03:19.134785Z","description","Minimal Web App to display links to multiple Toaster instances using Consul","is_private","last_updated","2021-06-02T19:34:42.623793Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[52],"application/vnd.docker.container.image.v1+json","name","pull_count",2338,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","# CI Scripts\n\nThis repository contains scripts and Docker Compose files that form\nthe Wind River Linux Continuous Integration Prototype.\n\n## Introduction\n\nI have built two build systems at WindRiver, but they were very\nspecific to the WRLinux workflow and internal infrastructure. This is\nmy third build system and an attempt to make a generic build and CI\nsystem with features around building and testing Yocto.\n\nThis repo contains the scripts to orchestrate all the components as\nwell as all the Dockerfile used to build the images. The images are\nhosted on Docker Hub and are automated builds linked to the\nDockerfiles in this repository. All the images with links to back to\nthe Dockerfile used to build them can be found here:\nhttps://hub.docker.com/r/windriver/\n\n## Features\n\n1) Multi-host builds using Docker Swarm. This provides an easy way to\nscale the machines in a build cluster up and down and Docker Swarm\nmakes this surprisingly simple.\n\n2) Developer builds. This enables build testing of patches before they\nare committed to the main branches. This leverages the [WR setup][1]\nprogram and a temporary layerindex to assemble a custom project that\nmatches the developer's local project.\n\n3) Toaster integration. A simple UI to dynamically expose the Toaster\ninterface of all builds in progress.\n\n## Getting Started\n\n### Requirements:\n\nmake\npython3: >= 3.3\n\nDocker CE: >= 17.03\n\nhttps://docs.docker.com/engine/installation/\n\nDocker Compose: >= 1.13.0\n\nhttps://docs.docker.com/compose/install/\n\n### Docker Socket permissions\n\nIn order for Jenkins Agent to start docker containers without opening\nthe socket to every user on the machine, the host system needs to\nallow a process with uid 1000 rw access to /var/run/docker.sock. This\ncan be done by adding uid 1000 to the docker group. Currently if the\nhost docker group has guid 995 to 999, this will enable access.\n\nIf the /var/run/docker.sock does not sufficient permissions, the swarm\nclient image will attempt to enable world rw permissions on the socket\nbefore attempting the build.\n\n### Single Host Setup\n\nTo start Jenkins Master and Agent on a single system using\ndocker-compose run:\n\n ./start_jenkins.sh\n\nThis will download the images from the Docker Cloud/Hub and start the\nimages using docker-compose.\n\nThe Jenkins web UI is accessible at https://localhost/jenkins. If\nattempting to access the web UI from a different machine, replace\nlocalhost with the name or IP of the server where the repository was\ncloned to.\n\nThe Jenkins interface is behind an nginx reverse proxy which uses a\nself signed certificate to provide TLS. The browser will warn you that\nthe web page is using a cert from an unknown CA and require you to\ngrant a security exception.\n\n### Multi-Host Setup\n\nThe CI prototype supports distributing builds onto multiple machines\nusing Docker Swarm.\n\n ./start_jenkins.sh --swarm\n\nThe machine where the `start_jenkins.sh` script is run must be a\nDocker swarm manager node. It will be labeled the \"master\" node and\nno builds will be scheduled on this node.\n\nUsing Docker Swarm requires some manual setup on each machine that\nwill be part of the cluster. Each node needs Docker 17.03+\ninstalled. On the node that will be master run:\n\n docker swarm init\n\nand note the provided command line with join token. On the worker nodes\nrun the provided command which will have the following form:\n\n docker swarm join --token :2377\n\nLimitations:\n\n1. Each worker node will start one Jenkins worker with 2\n executors. I will investigate using Docker labels to set Jenkins\n Node labels to control number of executors and job scheduling\n control.\n2. No scaling tests have been performed, although I expect this setup\n to work well for a 2-10 machine cluster.\n\nFor more information on managing a Docker Swarm, consult\nthe [Docker Swarm Documentation][2]\n\n[2]: https://docs.docker.com/engine/swarm/\n\n\n## Scheduling Jobs\n\nBy default the Jenkins Master does not have authentication enabled and\njobs can be submitted from any machine. On the same or a different\nmachine, clone this repository. To install the python-jenkins package\nlocally and submit a job run:\n\n make setup\n .venv/bin/python3 ./jenkins_job_submit.py \\\n --jenkins --configs \\\n --build_configs_file configs/WRLinux9/combos-WRLINUX_9_BASE.yaml\n\nThis will install all the python libraries into .venv and contact\nthe Jenkins Master and schedule a build on the Jenkins Agent. The\n is a comma separated list of names from the\nyaml file. As examples the combos-WRLINUX_9_BASE.yaml contains names\nsuch as qemuarm_glibc-small_Base and qemux86-64_glibc-core_Base.\n\nThe combos-WRLINUX_9_BASE.yaml file is a generated list of valid\ncombinations of qemu bsps and configuration options. At WindRiver we\ngenerate these yaml files using the machines and images listed in the\nLayerIndex.\n\n### Scheduling Poky Builds\n\nAn example config is provided to demonstrate building Poky locally:\n\n .venv/bin/python3 ./jenkins_job_submit.py --jenkins \\\n --build_configs_file configs/OpenEmbedded/combos-pyro.yaml \\\n --configs=pyro-minimal\n\nTo reuse the ubuntu1604_64 image, the poky build uses the WRLinux\nbuildtools from Github.\n\n### Toaster Integration\n\nAll builds enable toaster by default and the prototype uses\nRegistrator and Consul to discover toaster instances. The toaster\naggregator webapp provides an overview of running toaster instances\nand the current progress of each as well as links to the individual\ntoaster instances.\n\nThe toaster aggregator web UI is available at\nhttps://localhost/toaster_aggregator\n\n### Post build operations\n\nThe conventional way to add post build operations on Jenkins Pipeline\nprojects is to add stages to the pipeline. Each pipeline stage would\nalso require more build parameters be added to the job\nconfiguration. Each additional post build step would require\nsignificant changes and would not compose well.\n\nThe post build operations can be run in a different container than the\nbuild to avoid having to add tools like rsync to the build\ncontainer. This also allows the build to run without network\naccess. To select a different post build image:\n\n .venv/bin/python3 ./jenkins_job_submit.py \\\n --jenkins --postprocess_image \n\nThe prototype contains a generic post build step that does not require\nmodifications to job config or Jenkinsfile. The post build scripts are\nlocated in the scripts directory and can be selected to run using the\ncommand line:\n\n .venv/bin/python3 ./jenkins_job_submit.py \\\n --jenkins --post_success=rsync,cleanup \\\n --post_fail=send_email,cleanup,report\n\nThis would run the scripts/rsync.sh and scripts/cleanup.sh scripts\nafter a successful build and scripts/send_email.sh, scripts/cleanup.sh\nand scripts/report.sh after a failed build.\n\nTo pass parameters to the post build scripts use:\n\n .venv/bin/python3 ./jenkins_job_submit.py \\\n --jenkins --postprocess_args FOO1=bar,FOO2=baz\n\nThe prototype will take the parameters, split them and inject them\ninto the postbuild environment.\n\nTo add post build steps, add a script to the scripts directory and add\nthe script name to the --post_success and/or --post_fail and add any\nrequired parameters to --postprocess_args.\n\n## Developer Builds\n\nAn ideal Continuous Integration workflow supports the testing of\npatches before they are committed to the \"trunk\" branches. Also known\nas pre-merge testing or pull request testing. The standard git\nworkflow is to use topic branches and have the CI system run tests\nagainst the topic branch.\n\nYocto projects contain a hierarchy of git repositories and there isn't\na standardized way to create a project area. The wr-lx-setup [1]\nattempts to standardize creation of Yocto projects using the\nLayerindex as a source of available layers and locations. This\nsimplifies and standardizes the setup of Yocto projects, but it adds\nthe Layerindex as a component required by the CI workflow.\n\nThe prototype supports creation of a temporary per build Layerindex\nand modifying the attributes of a layer in the temporary\nLayerindex. This enables a developer to create a topic branch on a\nlayer git repository and run builds and tests using this branch.\n\nAn example workflow:\n\n .venv/bin/python3 ./jenkins_job_submit.py \\\n --jenkins --configs master-minimal \\\n --devbuild_layer_name openembedded-core \\\n --build_configs_file configs/OpenEmbedded/combos-master.yaml \\\n --configs master-minimal --devbuild_layer_name openembedded-core \\\n --devbuild_layer_vcs_url git://github.com/kscherer/openembedded-core.git \\\n --devbuild_actual_branch devbuild\n\nThe sequence of events is:\n\n1. Temporary Layerindex is created and retrieves master branch info\n from official layers.openembedded.org Layerindex.\n2. The vcs_url and actual_branch for the openembedded-core layer in\n temp layerindex is changed and the temp layerindex runs update.py\n to parse this layer. If the update fails, the build also\n fails. This provides a mechanism to test the addition of new\n layers.\n3. The wr-lx-setup.sh program is run using the temp layerindex as its\n source and creates a build area using the openembedded layer as\n defined in the supplied vcs_url.\n4. After setup is complete, the normal build process continues.\n5. After build is complete, the temp layerindex is shutdown and\n cleaned up.\n\nCurrent Limitations:\n\n1. The Layerindex assumes that bitbake and openembedded-core\n repositories are located on the same git server at the same path.\n2. Only changing a single layer is currently supported. There is no\n technical reason why multiple layers could not be changed.\n3. Only tested with http://layers.openembedded.org\n\n[1]: https://github.com/Wind-River/wr-lx-setup\n\n## Rsync server\n\nTo support collection of build artifact results from a build cluster,\nan rsync server has been integrated. The runs beside the reverse proxy\nservice and accepts files without authentication. The postbuild image\nnow runs in the same network as the rsync server and can use the\npostbuild rsync script to copy files to this server or any external\nserver.\n\nThe contents of the rsync server are available over HTTPS through the\nreverse proxy at `https:///builds`\n\n## Using Jenkins Credential Store to access Git server\n\nJenkins has an encrypted credential store which can manage credentials\nused to access the git server. Connect to Jenkins and select\nCredentials in the left menu. Then select System -> \"Global\n(unrestricted)\" -> Add Credentials. Use ID \"git\" which is the default\nused by jenkins_job_submit.py.\n\n .venv/bin/python3 ./jenkins_job_submit.py \\\n --jenkins --configs \\\n --build_configs_file configs/WRLinux9/combos-WRLINUX_9_BASE.yaml \\\n --git_credential=enable\n\n## Scheduling Jobs when Jenkins requires Authentication\n\nThe jenkins_job_submit.py can submit jobs to the Jenkins master that\nrequires Authentication. If unauthenticated access fails,\njenkins_job_submit.py check for a local file containing the\nauthentication credentials. The default name of the local auth file is\njenkins_auth.txt at the root path of this CI script. To use a\ndifferent file name use the command line option\n`--jenkins-auth=FILE_NAME`. Format of the local auth file\nshould be `USERNAME:API_TOKEN`, and only one line of text is allowed.\n\nThe windriver/jenkins-master image requires authentication and has a\nspecial mechanism to retrieve this information transparently. If\nauthentication to a different Jenkins server fails, contact the\nmanager of Jenkins server and put the valid authentication in local\nauth file to submit jobs.\n\n .venv/bin/python3 ./jenkins_job_submit.py \\\n --jenkins --configs \\\n --build_configs_file configs/WRLinux9/combos-WRLINUX_9_BASE.yaml \\\n --jenkins_auth \n\n## Runtime tests\n\nRuntime tests are supported using LAVA, which is a server running\noutside of CI scripts. The LAVA server configuration is set in\nconfig files which locates in configs folder.\n\nTo enable runtime tests, the following settings in the YAML config\nfile under configs folder are required:\n\n```yaml\n post_build:\n post_process_image: postbuild\n postprocess_args:\n RSYNC_SERVER: yow-lpdtest.wrs.com\n RSYNC_DEST_DIR: builds/wrlinux10\n HTTP_ROOT: http://yow-lpdtest.wrs.com/tftpboot\n REPORT_SERVER: http://yow-lpdtest.wrs.com:9200\n SMTPSERVER: prod-webmail.windriver.com\n EMAIL: first.last@windriver.com\n post_success:\n - rsync\n - cleanup\n post_fail:\n - rsync\n - cleanup\n - send_mail\n - report\n\n test_config:\n\ttest: [disable (default) or test_suite_name]\n test_image: postbuild\n test_args:\n LAVA_SERVER: \n LAVA_USER: \n NFS_ROOT: /net/yow-lpdtest/var/lib/tftpboot\n TEST_DEVICE: [simics (default) or hardware]\n RETRY: 1\n```\n\n### Test Suites\nIn above YAML config file, **test** under **test_config** section is used to set which test suite\nwill be used for runtime test.\n\n_Supported Runtime Tests_\n\n| Product | Supported Test Suite | Supported Device |\n|--------------|---------------------------------|--------------------|\n| WRLinux 9 | oeqa-default-test (default) | simics or hardware |\n| WRLinux 10 | oeqa-default-test (default) | simics or hardware |\n| WRLinux 10 | linaro-smoke-test | simics or hardware |\n| WRLinux 10 | linaro-busybox-test | simics or hardware |\n| WRLinux 10 | linaro-signal-test | simics or hardware |\n| WRLinux 10 | linaro-singlenode-advanced-test | simics or hardware |\n| WRLinux 10 | linaro-pi-stress-test | hardware |\n| WRLinux 10 | linaro-pmq-test | hardware |\n\nConfigurations for each test suite are set in test_configs.yaml under configs folder. \nHere is and example:\n\n```yaml\n- name: oeqa-default-test\n prebuild_cmd_for_test:\n - test_configure.py\n build_options:\n - INHERIT += \"testexport\"\n - IMAGE_INSTALL_append += \"python3-pkgutil\"\n - IMAGE_INSTALL_append += \"python3-unittest\"\n - IMAGE_INSTALL_append += \"python3-multiprocessing\"\n - TEST_TARGET_IP = \"localhost\"\n - TEST_SERVER_IP = \"localhost\"\n - TEST_SUITES = \"ping ssh df connman syslog xorg scp vnc date pam perl python rpm ldd smart dmesg dash\"\n lava_test_repo: git://ala-lxgit.wrs.com/lpd-ops/lava-test.git\n simics:\n job_template: lava-test/jobs/templates/wrlinux-10/x86_simics_job_oeqa-default-test_template.yaml\n timeout: 420\n hardware:\n job_template: lava-test/jobs/templates/wrlinux-10/x86_64_job_oeqa-default-test_template.yaml\n timeout: 480\n```\n\n## Modifying docker images\n\nThe CI prototype uses the following images:\n\n- windriver/jenkins-master\n- windriver/jenkins-swarm-client\n- windriver/ubuntu1804_64\n- windriver/toaster_aggregator\n- blacklabelops/nginx\n- gliderlabs/registrator\n- consul\n- windriver/layerindex\n\nTo test image modifications rebuild the container locally and run:\n\n ./start_jenkins.sh --no-pull\n\n## TODO\n\n- Build notifications\n- Simplify settings of runtime tests\n- Select a good project name\n\n## Contributing\n\nContributions submitted must be signed off under the terms of the Linux\nFoundation Developer's Certificate of Origin version 1.1. Please refer to:\n https://developercertificate.org\n\nTo submit a patch:\n\n- Open a Pull Request on the GitHub project\n- Optionally create a GitHub Issue describing the issue addressed by the patch\n\n# Docker Images\n\nThis repository contains only the Dockerfiles used to generate the\nimages. The images are assembled and hosted by Docker on the Docker\nCloud/Hub.\n\nThe images contain Open Source software as distributed by the\nfollowing projects.\n\n- Ubuntu Linux: https://www.ubuntu.com/\n- Alpine Linux: https://www.alpinelinux.org/\n- Jenkins: https://jenkins.io/\n- Jenkins Plugins: https://plugins.jenkins.io/\n- Docker: https://www.docker.com/\n\n# Included Third Party Components\n\n## MIT License\n\n- jQuery Compat JavaScript Library v3.0.0-pre 81b6e46522d8c680f6c38d5e95c732b2b47130b9\n- Skeleton V2.0.4\n- DataTables 1.10.13\n\n# License\n\nMIT License\n\nCopyright (c) 2017 Wind River Systems Inc.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n","has_starred","hub_user","immutable_tags_settings",{"_71":11,"_72":73},"enabled","rules",[74],".*","is_automated","permissions",{"_78":17,"_79":11,"_80":11},"read","write","admin","user","storage_size",351117408,"orgData",{"_86":87,"_88":89,"_90":91,"_92":29,"_93":94,"_95":96,"_97":98,"_99":100,"_101":102,"_103":94,"_104":94,"_105":94,"_106":17},"id","688daead8b8c421299d25bf1838badda","uuid","688daead-8b8c-4212-99d2-5bf1838badda","type","organization","orgname","full_name","","location","Alameda, California","company","Wind River Systems","profile_url","http://windriver.com","date_joined","2017-06-21T18:46:37.471735Z","gravatar_url","gravatar_email","badge","is_active","recentTags",[109],{"_110":40,"_111":112,"_113":114,"_86":115,"_116":117,"_46":134,"_135":112,"_136":29,"_137":52,"_53":138,"_139":140,"_141":125,"_142":134,"_143":62,"_144":17},"content_type","creator",1862962,"full_size",23882242,15955001,"images",[118],{"_119":120,"_121":122,"_123":94,"_124":125,"_126":127,"_128":129,"_130":94,"_131":-5,"_132":114,"_59":62,"_133":-5},"architecture","amd64","digest","sha256:cc9c14b699207858b5a5ae8745de7d38f65185eac16481f9cd5ecaeb89baaa2c","features","last_pulled","2026-09-22T09:58:10.602810058Z","last_pushed","2021-06-02T19:34:43Z","os","linux","os_features","os_version","size","variant","2021-06-02T19:34:42.300112Z","last_updater","last_updater_username","media_type","latest","repository",1867927,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_34":156,"_163":164,"_152":173,"_174":11},{"_36":157,"_38":158,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":159,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":160,"_75":17,"_76":162,"_81":29,"_82":83},[],[40],[52],{"_71":11,"_72":161},[74],{"_78":17,"_79":11,"_80":11},"tags",{"_165":60,"_166":-5,"_167":-5,"_168":169},"count","next","previous","results",[170],{"_110":40,"_111":112,"_113":114,"_86":115,"_116":171,"_46":134,"_135":112,"_136":29,"_137":52,"_53":138,"_139":140,"_141":125,"_142":134,"_143":62,"_144":17},[172],{"_119":120,"_121":122,"_123":94,"_124":125,"_126":127,"_128":129,"_130":94,"_131":-5,"_132":114,"_59":62,"_133":-5},[],"canDeleteTags","actionData","errors"]

windriver/toaster_aggregator

Tags for windriver/toaster_aggregator