ricmathie/python_java repository overview

# Secret for root:password enter password you want and ctrl+D to exit 
docker secret create storm_password -


# we need zookeeper, ZOO_SERVERS env just appends to zoo.conf so we can get the auto purge to work in that way
docker service create --name zookeeper \
  --network mynet \
  -p 2181:2181 \
  -e "ZOO_SERVERS=autopurge.purgeInterval=12" \
  zookeeper:3.4.9


docker service create --name nimbus \
  --network mynet \
  -e 'ZK_PORT_2181_TCP_ADDR=- zookeeper' \
  -e 'NIMBUS_HOSTS=tasks.nimbus' \
  -e 'HOST_INTERFACE=eth2' \
  --publish "3773:3773" \
  --publish "3772:3772" \
  --publish "6627:6627" \
  --publish "49082:8000" \
  --secret=storm_password \
  ricmathie/storm-nimbus:0.0.2

# setup venvs and log folder for streamparse
docker secret create streamparse_requirements requirements.txt

docker service create -t --name storm_venv \
  --mode global \
  --mount type=bind,dst=/mount,src=/home \
  --secret=streamparse_requirements \
  ricmathie/storm-supervisor:0.0.2 \
  bash -c "mkdir -p /mount/streamparse/logs ;\
      mkdir -p /mount/streamparse/virtualenvs ; \
      cd /mount/streamparse/ ;\
      virtualenv virtualenvs --system-site-packages ;\
      . virtualenvs/bin/activate ;\
      cat /run/secrets/streamparse_requirements ;\
      pip install -r /run/secrets/streamparse_requirements; sh"

# give enough time for nimbus to achive running state
sleep 100

# nslookup tasks.nimbus | awk '/Address: / {print $2}' | nslookup | awk '/name = / {print substr($4, 1, length($4)-1)}'

docker service create --name storm_supervisor \
  --mode global \
  --network mynet \
    --mount type=bind,dst=/mount/streamparse,src=/home/streamparse \
  -e 'ZK_PORT_2181_TCP_ADDR=- zookeeper' \
  -e 'NIMBUS_HOSTS=tasks.nimbus' \
  -e 'HOST_INTERFACE=eth2' \
  --publish "6703:6703" \
  --publish "49080:8000" \
  --secret=storm_password \
  ricmathie/storm-supervisor:0.0.2

docker service create --name storm_ui \
  --network mercury \
  --constraint 'node.role == manager' \
  -e 'ZK_PORT_2181_TCP_ADDR=- zookeeper' \
  -e 'NIMBUS_HOSTS=tasks.nimbus' \
  -e 'HOST_INTERFACE=eth2' \
  --publish "49081:8080" \
  --secret=storm_password \
  ricmathie/storm-ui:0.0.2

# check venv sucseeded
docker service logs storm_venv
docker service rm storm_venv

# hostname -i doesn't always grab the ip of the network you want so you can set some commands to get the correct advertised ip address
HOST_INTERFACE # default host interface to grab ip
HOST_COMMAND   # command to return the Host ip

ZOOKEEPER_HOSTS="tasks.zookeeper.mercury"}
ZOOKEEPER_COMMAND="hosts_to_yaml $ZOOKEEPER_HOSTS"}

NIMBUS_HOSTS="tasks.nimbus.mercury"}
NIMBUS_COMMAND="hosts_to_yaml $NIMBUS_HOSTS"}
DRPC_COMMAND="hosts_to_yaml $NIMBUS_HOSTS"}

STORM_PASSWORD  # used to set Storm SSH password leave unset for docker secrets

ZK_PORT_2181_TCP_ADDR     # used to set ZOOKEEPER_IP leave unset to use eval $ZOOKEEPER_COMMAND
NIMBUS_PORT_6627_TCP_ADDR # used to set NIMBUS_IPs leave unset to use eval $NIMBUS_COMMAND
DRPC_PORT_6627_TCP_ADDR   # used to set DRPC_IP leave unset to use eval $DRPC_COMMAND