[{"_1":2,"_182":-5,"_183":-5},"loaderData",{"_3":4,"_17":18,"_25":26,"_161":162},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7,"_13":14,"_15":16},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","theme","system","contrast","normal","routes/_layout",{"_19":-5,"_20":21,"_22":21,"_23":11,"_24":21},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_27":28,"_29":-5,"_30":11,"_31":11,"_32":33,"_34":35,"_36":-5,"_37":11,"_38":39,"_88":89,"_111":112,"_149":-5,"_150":11,"_151":21,"_152":11,"_153":11,"_154":-5,"_155":11,"_156":-5,"_157":158,"_159":113,"_160":-7},"canonicalUrl","https://hub.docker.com/r/datastrophic/mesos-spark-notebook/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","datastrophic","reponame","mesos-spark-notebook","cloudBuildSource","isAutobuildsAvailable","repo",{"_40":41,"_42":43,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":55,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":62,"_64":65,"_66":-5,"_67":68,"_69":70,"_71":11,"_72":33,"_73":74,"_79":21,"_80":81,"_85":33,"_86":87},"categories",[],"content_types",[44],"image","date_registered","2017-02-20T12:36:01.649682Z","description","Spark-notebook on top of datastrophic/mesos-spark container","is_private","last_updated","2017-02-20T12:43:27.606481Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[56],"application/vnd.docker.container.image.v1+json","name","pull_count",136,"repository_type","star_count",1,"status","status_description","active","source","collaborator_count",0,"full_description","# Mesos Docker Containers\n\n## Overview\nSet of Mesos-related docker containers built to address the latest updates of Mesos frameworks. Base Mesos images are used to install \nChronos, Spark and Kafka binaries on top of them and run framework containers on bare metal Mesos installations. For demonstration purposes\n there's a docker-compose file provided to run cluster locally, however, limitations exist for local setup.\n\nAll images use Ubuntu 16.04 by default, framework images derive from Mesos base with Oracle Java 8 installed, Mesos agent images \nderive from Mesos base with Docker installed.\n\n## Running Spark on Mesos\nFrom any of Mesos agent nodes with Docker installed:\n\nspark-shell:\n\n docker run -ti --net=host datastrophic/mesos-spark:mesos-1.1.0-spark-2.1.0 \\\n spark-shell --master mesos://master.mesos:5050 --conf spark.mesos.executor.docker.image=datastrophic/mesos-spark:mesos-1.1.0-spark-2.1.0\n\nspark-submit:\n\n docker run -ti --net=host datastrophic/mesos-spark:mesos-1.1.0-spark-2.1.0 \\\n spark-submit --master mesos://master.mesos:5050 --conf spark.mesos.executor.docker.image=datastrophic/mesos-spark:mesos-1.1.0-spark-2.1.0 \\\n --class org.apache.spark.examples.SparkPi /spark/examples/jars/spark-examples_2.11-2.1.0.jar 250\n\nSubmitting Spark job to Chronos to be executed on a regular manner:\n\n curl -L -H 'Content-Type: application/json' -X POST http://chronos.marathon.mesos:4400/v1/scheduler/iso8601 -d '{\n \"schedule\": \"R/2017-02-14T12:00:00.000Z/PT3M\",\n \"name\": \"scheduled-spark-submit\",\n \"container\": {\n \"type\": \"DOCKER\",\n \"image\": \"datastrophic/mesos-spark:mesos-1.1.0-spark-2.1.0\",\n \"network\": \"HOST\"\n },\n \"cpus\": \"0.5\",\n \"mem\": \"1024\",\n \"fetch\": [],\n \"command\": \"spark-submit --master mesos://master.mesos:5050 --conf spark.mesos.executor.docker.image=datastrophic/mesos-spark:mesos-1.1.0-spark-2.1.0 --class org.apache.spark.examples.SparkPi /spark/examples/jars/spark-examples_2.11-2.1.0.jar 250\"\n }'\n\n## Running Chronos on Mesos\nFrom any of Mesos agent nodes with Docker installed:\n\n docker run -ti \\\n --net=host -p 4400:4400 \\\n -e CHRONOS_HTTP_PORT=4400 \\\n -e CHRONOS_MASTER=zk://:2181,:2181,:2181/mesos \\\n -e CHRONOS_ZK_HOSTS=:2181,:2181,:2181 \\\n datastrophic/chronos:mesos-1.1.0-chronos-3.0.1\n\nSubmitting to Marathon:\n\n curl -XPOST 'http://marathon.mesos:8090/v2/apps' -H 'Content-Type: application/json' -d '{\n \"id\": \"chronos\",\n \"container\": {\n \"type\": \"DOCKER\",\n \"docker\": {\n \"network\": \"HOST\",\n \"image\": \"datastrophic/chronos:mesos-1.1.0-chronos-3.0.1\"\n }\n },\n \"env\": {\n \"CHRONOS_HTTP_PORT\":\"4400\",\n \"CHRONOS_MASTER\":\"zk://:2181,:2181,:2181/mesos\",\n \"CHRONOS_ZK_HOSTS\":\":2181,:2181,:2181\"\n },\n \"ports\": [4400],\n \"cpus\": 1,\n \"mem\": 512,\n \"instances\": 1,\n \"constraints\": [[\"hostname\", \"UNIQUE\"]]\n }'\n\n## Running Mesos locally\nThis setup is more for development and educational purposes and hits its limits when it comes to running docker containers via Marathon.\n\n###docker-compose.yaml reference\n```\nversion: '3'\nservices:\n\n zookeeper:\n image: mesoscloud/zookeeper:3.4.6-ubuntu-14.04\n hostname: \"zookeeper\"\n container_name: zookeeper\n ports:\n - \"2181:2181\"\n - \"2888:2888\"\n - \"3888:3888\"\n\n mesos-master:\n image: datastrophic/mesos-master:1.1.0\n hostname: \"mesos-master\"\n container_name: master\n privileged: true\n environment:\n - MESOS_HOSTNAME=mesos-master\n - MESOS_CLUSTER=SMACK\n - MESOS_QUORUM=1\n - MESOS_ZK=zk://zookeeper:2181/mesos\n - MESOS_LOG_DIR=/tmp/mesos/logs\n links:\n - zookeeper\n ports:\n - \"5050:5050\"\n\n mesos-slave:\n image: datastrophic/mesos-slave:1.1.0\n hostname: \"mesos-slave\"\n container_name: slave\n privileged: true\n environment:\n - MESOS_HOSTNAME=mesos-slave\n - MESOS_PORT=5151\n - MESOS_MASTER=zk://zookeeper:2181/mesos\n links:\n - zookeeper\n - mesos-master\n ports:\n - \"5151:5151\"\n volumes:\n - /var/run/docker.sock:/var/run/docker.sock\n\n marathon:\n image: datastrophic/marathon:1.3.6\n hostname: \"marathon\"\n container_name: marathon\n environment:\n - MARATHON_HOSTNAME=marathon\n - MARATHON_MASTER=zk://zookeeper:2181/mesos\n - MARATHON_ZK=zk://zookeeper:2181/marathon\n links:\n - zookeeper\n - mesos-master\n ports:\n - \"8080:8080\"\n\n chronos:\n image: datastrophic/chronos:mesos-1.1.0-chronos-3.0.1\n hostname: \"chronos\"\n container_name: chronos\n environment:\n - CHRONOS_HTTP_PORT=4400\n - CHRONOS_MASTER=zk://zookeeper:2181/mesos\n - CHRONOS_ZK_HOSTS=zookeeper:2181\n links:\n - zookeeper\n - mesos-master\n ports:\n - \"4400:4400\"\n```\n\n###Environment\nCurrent steps are tested with Docker for Mac:\n\n Docker version 1.13.1\n docker-compose version 1.11.1\n docker-py version 2.0.2\n\nContainers configuration is located in [docker-compose.yml](docker-compose.yml) in the root folder of this repo.\n\nTo rebuild images please refer to [bin/build-all.sh](bin/build-all.sh) helper script\n\n###Running Mesos cluster\nTo spin up a cluster with zookeeper, master, agent, marathon and chronos one instance each:\n \n docker-compose -p mesos up -d --force-recreate\n \n (!) It might be useful to add alias for containers hostnames to /etc/hosts \n 127.0.0.1 mesos-master mesos-slave zookeeper marathon chronos \n \nIt is important to specify `-p mesos` flag (docker-compose project name) to ease the reference to default network created automatically.\n\nAfter this Mesos Master and Slave, Marathon and Chronos should be available:\n\n* Mesos Master [http://mesos-master:5050](http://mesos-master:5050)\n* Marathon [http://marathon:8080](http://marathon:8080)\n* Chronos [http://chronos:4400](http://chronos:4400)\n\nRemove the containers with:\n \n docker-compose -p mesos down && docker-compose -p mesos rm -f\n \n###Using multiple agents locally\nMultiple Mesos agents could be used in local setup, for this entries in `docker-compose.yml` should be duplicated with \nappropriate port remapping to avoid conflicts. Check out [docker-compose-bare-mesos.yml](docker-compose-bare-mesos.yml) \nfor the reference.\n\n###Docker in Docker\nSocket binding allows to run docker containers from within Mesos agents which have Docker installed:\n \n volumes:\n - /var/run/docker.sock:/var/run/docker.sock\n \nHere is an example of running dockerized Chronos inside Mesos Docker container:\n\n- ensure current docker-compose stack is not running `docker-compose -p mesos top` and stop it if it is `docker-compose -p mesos down && docker-compose -p mesos rm -f`\n- run docker-compose with dedicated compose file containing zookeper, Mesos master and two agents with `docker-compose -f docker-compose-bare-mesos.yml -p mesos up -d --force-recreate`\n- attach to one of the running agents `docker exec -ti slave-1 /bin/bash`\n- run \n\n\n docker run -ti \\\n --net=mesos_default -p 4400:4400 \\\n -e CHRONOS_HTTP_PORT=4400 \\\n -e CHRONOS_MASTER=zk://zookeeper:2181/mesos \\\n -e CHRONOS_ZK_HOSTS=zookeeper:2181 \\\n datastrophic/chronos:mesos-1.1.0-chronos-3.0.1\n\n- now Chronos UI should be available at [http://chronos:4400](http://chronos:4400) or [http://localhost:4400](http://localhost:4400) if you haven't modified `/etc/hosts`\n- shutdown the stack `docker-compose -p mesos -f docker-compose-bare-mesos.yml down && docker-compose -p mesos -f docker-compose-bare-mesos.yml rm -f`\n\n\nThe approach used here is in specifying network `mesos_default` from parent containers so all the top-level containers could be reached and \ntheir hostnames resolved. This network is created by default when docker-compose is executed for first time.\n\n###Limitations of local setup\nIn production Mesos cluster the best practice is to run Chronos via Marathon to provide high availability guarantees. \nThis could be done by posting appropriate Marathon configuration for Chronos application:\n \n curl -XPOST 'http://marathon:8080/v2/apps' -H 'Content-Type: application/json' -d '{\n \"id\": \"chronos\",\n \"container\": {\n \"type\": \"DOCKER\",\n \"docker\": {\n \"network\": \"BRIDGE\",\n \"privileged\":true,\n \"image\": \"datastrophic/chronos:mesos-1.1.0-chronos-3.0.1\",\n \"parameters\": [\n { \"key\": \"env\", \"value\": \"CHRONOS_HTTP_PORT=4400\" },\n { \"key\": \"env\", \"value\": \"CHRONOS_MASTER=zk://zookeeper:2181/mesos\" },\n { \"key\": \"env\", \"value\": \"CHRONOS_ZK_HOSTS=zookeeper:2181\"}\n ],\n \"portMappings\": [\n { \"containerPort\": 4400 }\n ]\n }\n },\n \"cpus\": 1,\n \"mem\": 512,\n \"instances\": 1\n }'\n \nHowever, this configuration will not work locally because Marathon doesn't support custom network types and with any other network type \nChronos container WILL NOT be able to reach zookeeper and bind Chronos to routable address. \n\nSame limitations apply for Spark and Kafka Docker containers as well if they're executed from within Docker.\n \n## Where to go from here\n\n* [Mesos Workshop: usage examples and sample framework implementation](https://github.com/datastrophic/mesos-workshop)\n* [Resource Allocation in Mesos: Dominant Resource Fairness blog post](http://datastrophic.io/resource-allocation-in-mesos-dominant-resource-fairness-explained/)","has_starred","hub_user","immutable_tags_settings",{"_75":11,"_76":77},"enabled","rules",[78],".*","is_automated","permissions",{"_82":21,"_83":11,"_84":11},"read","write","admin","user","storage_size",1368162471,"orgData",{"_90":91,"_92":93,"_94":95,"_96":33,"_97":98,"_99":100,"_101":102,"_103":104,"_105":106,"_107":98,"_108":98,"_109":98,"_110":21},"id","0c70d39fc4434cc6b1aca09f397cc634","uuid","0c70d39f-c443-4cc6-b1ac-a09f397cc634","type","organization","orgname","full_name","","location","Stockholm","company","Datastrophic","profile_url","http://datastrophic.io/","date_joined","2015-02-04T21:41:42.226737Z","gravatar_url","gravatar_email","badge","is_active","recentTags",[113],{"_114":44,"_115":116,"_117":118,"_90":119,"_120":121,"_50":138,"_139":116,"_140":33,"_141":56,"_57":142,"_143":144,"_145":129,"_146":138,"_147":65,"_148":21},"content_type","creator",210905,"full_size",1368150250,8313140,"images",[122],{"_123":124,"_125":126,"_127":98,"_128":129,"_130":131,"_132":133,"_134":98,"_135":-5,"_136":118,"_63":65,"_137":-5},"architecture","amd64","digest","sha256:99f602a0e16d3958d8f139245f15c31cc65224f5a31f72b0465500ff6af68ce8","features","last_pulled","2026-10-15T11:14:35.945064206Z","last_pushed","2017-02-20T12:43:27Z","os","linux","os_features","os_version","size","variant","2017-02-20T12:43:27.14434Z","last_updater","last_updater_username","media_type","test","repository",1266839,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxRunInCloud","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"selectedTag","attestationsUrl","routes/_layout.r.$namespace.$reponame.tags",{"_38":163,"_170":171,"_157":180,"_181":11},{"_40":164,"_42":165,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":166,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":62,"_64":65,"_66":-5,"_67":68,"_69":70,"_71":11,"_72":33,"_73":167,"_79":21,"_80":169,"_85":33,"_86":87},[],[44],[56],{"_75":11,"_76":168},[78],{"_82":21,"_83":11,"_84":11},"tags",{"_172":62,"_173":-5,"_174":-5,"_175":176},"count","next","previous","results",[177],{"_114":44,"_115":116,"_117":118,"_90":119,"_120":178,"_50":138,"_139":116,"_140":33,"_141":56,"_57":142,"_143":144,"_145":129,"_146":138,"_147":65,"_148":21},[179],{"_123":124,"_125":126,"_127":98,"_128":129,"_130":131,"_132":133,"_134":98,"_135":-5,"_136":118,"_63":65,"_137":-5},[],"canDeleteTags","actionData","errors"]

datastrophic/mesos-spark-notebook

Tags for datastrophic/mesos-spark-notebook