Sign inSign up

wpcloud/proxy

By wpcloud

•Updated over 9 years ago

Image
0

2.6K

wpcloud/proxy repository overview

⁠GCE Tags for Load Balancing

Define if a machine may have traffic proxied to it. HAProxy uses this list to determine if to add it to a service load balanciner.

  • allow-deployment-updater
  • allow-metadata-service
  • allow-cluster-healing
  • allow-cluster-health
  • allow-docker-api
⁠ToDo
⁠Status Endpoints
  • controller:18095/api/status/v1/haproxy - current linternal
  • app/api/status/v1/php-fpm
  • app/api/status/v1/nginx
⁠Fault Tolerance
  • Varnish crashes.
  • Varnish up but can not route to Router.
  • Router crashes.
  • HAProxy crashes.
  • HAProxy does not start. Can not find self
⁠WAF Index Setup
curl -XPUT https://www.wpcloud.io:[email protected]:19100/waf-v1 --data '{"settings":{"index":{"number_of_shards":"2","number_of_replicas":"0"}}}'
curl -XPUT https://www.wpcloud.io:[email protected]:19100/waf-v1/_mapping/log-v1/ -d @static/mappings/log-v1.json
curl -XPUT https://www.wpcloud.io:[email protected]:19100/waf-v1/_mapping/blocked-v1/ -d @static/mappings/blocked-v1.json
⁠Surrogate Keys
curl -XPUT http://yuma.wpcloud.io:9200/api.wpcloud.io/_mapping/surrogate-key-v1/ -d @static/mapping/surrogate-key-v1.json
⁠Starting Service
service pull wpcloud/proxy
git clone [email protected]:wpCloud/docker-proxy.git /opt/sources/wpCloud/docker-proxy
systemctl --force link /opt/sources/wpCloud/docker-proxy/static/proxy.service
systemctl --force reload-or-restart proxy.service
⁠Soft Rollout / Update
curl localhost:6081/api/status/v1/health.json
/usr/bin/docker --host=unix:///var/run/dockerServices.sock exec  proxy git fetch
/usr/bin/docker --host=unix:///var/run/dockerServices.sock exec  proxy git reset --hard HEAD
/usr/bin/docker --host=unix:///var/run/dockerServices.sock exec  proxy git pull
/usr/bin/docker --host=unix:///var/run/dockerServices.sock exec  proxy sudo proxy update-hosts --silent
/usr/bin/docker --host=unix:///var/run/dockerServices.sock exec  proxy bash varnish.check 
/usr/bin/docker --host=unix:///var/run/dockerServices.sock exec  proxy bash varnish.reload 
curl localhost:6081/api/status/v1/health.json
⁠Hard Rollout
service pull wpcloud/proxy:latest
systemctl restart proxy
/usr/bin/docker --host=unix:///var/run/dockerServices.sock exec  proxy sudo service varnish restart 
⁠Update Static Configs
curl api.wpcloud.io:2375/containers/json > /repositories/docker-proxy/etc/proxy/data/containers.json
⁠Polling Backend Health
bash -c "while true; do curl quantico.wpcloud.io:6081/api/status/v1/health.json; sleep 1; done"
⁠Polling Backend Health
while true; do web.curl www.saintolaf.org -I; sleep 1; done
while true; do web.curl api.wpcloud.io -H "host:penmarproperties.com" -I; sleep 1; done
bash -c "while true; do docker --host=unix:///var/run/dockerServices.sock exec -it proxy varnishadm backend.list UsabilityDynamics_Staging_Quantico; sleep 5; done"
bash -c "while true; do docker --host=unix:///var/run/dockerServices.sock exec -it proxy varnishadm backend.list DiscoDonniePresents_Production_Fallujah; sleep 2; done"
⁠Remove Admin
varnishadm -T amarah.wpcloud.io:6082 -S /etc/varnish/secret.key
varnishadm -T mosul.wpcloud.io:6082 -S /etc/varnish/secret.key
⁠Updating Deployed Service
git fetch && git reset --hard HEAD && git pull
echo "Varnish VLC is: $(sh /etc/varnish/bin/check.sh)."
echo "HAProxy: $(haproxy -f /etc/haproxy/haproxy.cfg -c)."
sudo service varnish restart
  • Albatross (controller.albatross) - AMQP monitor.
  • Interceptor (controller.interceptor) - Intelligent management request monitor.
⁠Logs
  • varnish.proxy - /var/log/varnish/varnish.proxy.log
  • varnish.log - /var/log/varnish/varnish.log

hatop -s /var/run/haproxy/haproxy.sock

⁠Varnish LOgging
varnishlog -g raw -i Backend_health
  0 Backend_health - Sampl_Site_Beta Went sick 4--X--- 0 1 1 0.000000 2.098419
  0 Backend_health - Sampl_Site_Beta Back healthy 4--X-RH 1 1 1 0.000324 1.573895 HTTP/1.1 200 OK           
varnishlog -g request -q "ReqUrl eq '/'" -i Timestamp,Begin,ReqMethod,ReqUrl,ReqHeader -d
varnishlog -q "ReqHeader ~ '^Host:www.ocoeecafe.com'"
varnishlog  -I RxHeader "Host:www.ocoeecafe.com"
⁠Usage
varnishadm backend.set_health DiscoDonniePresents_Staging_Quantico healthy
varnishadm backend.set_health UsabilityDynamics_Staging_Quantico healthy
varnishadm backend.set_health PrestigeEstateAgency_Staging_Ramadi healthy
varnishadm backend.set_health Sokhop_Production_Mosul sick
varnishadm backend.set_health Sokhop_Production_Mosul auto
curl -X PURGE localhost:6081/ -H "host:kibana.udx.io"
service exec controller wpc routes --test
service exec controller wpc routes
service exec controller sudo varnishd -C -f /etc/varnish/wpcloud-dynamic.vcl
service exec controller sudo service varnish reload
service exec controller varnishadm backend.list
⁠VCL Generation
  • app.cluster - A globally unique camcel-cased name used to group backends that will serve the application. App containers in a cluster (backends) run same branch of code.
  • app.hostname - This will be used for Docker Host and for Varnish backend checks, so request should return 200, not a redirection.
⁠Key Generation
openssl req -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt
⁠Testing

To use CURL to emulate a typical browser request:

function web.curl {
  curl $@ \
    -H 'Accept-Encoding: gzip, deflate, sdch' \
    -H 'Accept-Language: en-US,en;q=0.8' \
    -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2341.0 Safari/537.36'\
    -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' \
    -H 'Cache-Control: max-age=0' \
    -H 'Cookie: _ga=GA1.2.841805203.1425324333' \
    -H 'Connection: keep-alive' \
    -H 'x-access-token: yhcwokwserjzdjir'
}
⁠Environment Variables

sudo varnishd -C
-n /usr/local/var/varnish
-s malloc,1G
-f /Users/andy.potanin/Repositories/docker-controller/test/fixtures/varnish-restarts.vcl
-a 0.0.0.0:8090

sudo varnishd -F
-n /usr/local/var/varnish
-f /Users/andy.potanin/Repositories/docker-controller/test/fixtures/varnish-restarts.vcl
-a 0.0.0.0:8090

⁠Container ElasticSearch Object
  • controller - Hostname of controller that owns the service.
  • container.State.Running - true|false
  • container.State.Pid - Pid of container, if running.
⁠Actions
  acl abuser src,map_ip_int(abusers.lst,0) -m int eq 1
  http-request tarpit if abuser

Block IP

   echo "add map abusers.lst 10.0.3.7 1" | socat - unix:/tmp/haproxy

Unblock IP

   echo "del map abusers.lst 10.0.3.7" | socat - unix:/tmp/haproxy

Reload HAP

  haproxy -f /etc/haproxy/haproxy.cfg -c && sudo haproxy -f /etc/haproxy/haproxy.cfg -sf $(pidof haproxy)
⁠Development
/usr/bin/docker --host=unix:///var/run/dockerServices.sock run -it --rm \
  --name=proxy.dev \
  --net=host \
  --memory=16g \
  --privileged=true \
  --volume=/etc/hosts:/etc/hosts:rw \
  --volume=/opt/storage:/opt/storage \
  --volume=/opt/sources/wpCloud/docker-proxy:/opt/sources/wpCloud/docker-proxy \
  --env=COREOS_PUBLIC_IPV4=${COREOS_PUBLIC_IPV4} \
  --env=COREOS_PRIVATE_IPV4=${COREOS_PRIVATE_IPV4} \
  --entrypoint="/bin/bash" \
  wpcloud/proxy
⁠Proxy Testing
curl 'http://127.0.0.1:12050/index.php' \
  -H "Host:www.usabilitydynamics.com" \
  -H "x-set-branch:staging" \
  -H 'Accept-Encoding: gzip, deflate, sdch' \
  -H 'Accept-Language: en-US,en;q=0.8' \
  -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36' \
  -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' \
  -H 'Referer: http://ci.usabilitydynamics.com/wp-admin/network/index.php' \
  -H 'Cookie: wordpress_740e7175421faec660020407b29a2af0=andy%7C1444207375%7CTCgj9pMiaF8zsa2qlhuiGZwZL49QrPpprQjPq1sA0b1%7C6383ece8fc80191ad1d449bde86ed7ca75a31e0df9648b37a6567addc6f1a320; wordpress_test_cookie=WP+Cookie+check; wordpress_logged_in_740e7175421faec660020407b29a2af0=andy%7C1444207375%7CTCgj9pMiaF8zsa2qlhuiGZwZL49QrPpprQjPq1sA0b1%7C53518254a7da3cc75b81b6bd04fb1a6a52316e83c827e280da519edb663ffa63; wp-settings-time-8=1444035366' \
  -H 'Connection: keep-alive' \
  -H 'Cache-Control: no-cache'

Tag summary

Content type

Image

Digest

sha256:a9ca53d7f…

Size

436.7 MB

Last updated

over 9 years ago

docker pull wpcloud/proxy