From ea04c0fe440a20c141df6db691f0e4082b467ffa Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Thu, 7 Dec 2017 15:24:13 -0800 Subject: [PATCH 01/32] Resolve #23 by adding CONSUL_DATACENTER_NAME env --- bin/consul-manage | 3 +++ etc/consul.hcl | 1 + local-compose.yml | 1 + 3 files changed, 5 insertions(+) diff --git a/bin/consul-manage b/bin/consul-manage index c66de4d..b43028b 100755 --- a/bin/consul-manage +++ b/bin/consul-manage @@ -8,6 +8,9 @@ set -eo pipefail preStart() { _log "Updating consul advertise address" sed -i "s/CONTAINERPILOT_CONSUL_IP/${CONTAINERPILOT_CONSUL_IP}/" /etc/consul/consul.hcl + + _log "Updating consul datacenter name" + sed -i "s/CONSUL_DATACENTER_NAME/${CONSUL_DATACENTER_NAME:-dc1}/" /etc/consul/consul.hcl } # diff --git a/etc/consul.hcl b/etc/consul.hcl index c9b5909..c8a0330 100644 --- a/etc/consul.hcl +++ b/etc/consul.hcl @@ -1,4 +1,5 @@ bind_addr = "CONTAINERPILOT_CONSUL_IP" +datacenter = "CONSUL_DATACENTER_NAME" data_dir = "/data" client_addr = "0.0.0.0" ports { diff --git a/local-compose.yml b/local-compose.yml index 1bb83fb..f3c21ba 100644 --- a/local-compose.yml +++ b/local-compose.yml @@ -15,5 +15,6 @@ services: - 8500 environment: - CONSUL=consul + - CONSUL_DATACENTER_NAME=dc1 command: > /usr/local/bin/containerpilot From cc21275e3ffa1002f1cc921c3ec8cd9f69fc8752 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Thu, 7 Dec 2017 16:07:03 -0800 Subject: [PATCH 02/32] Mention the environment variable in the README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 400ebe6..eb36599 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,10 @@ services: In our experience, including a Consul cluster within a project's `docker-compose.yml` can help developers understand and test how a service should be discovered and registered within a wider infrastructure context. +#### Environment Variables + +- `CONSUL_DATACENTER_NAME`: sets the name of the data center in which the Consul cluster is running. + ### Clients ContainerPilot utilizes Consul's [HTTP Agent API](https://www.consul.io/api/agent.html) for a handful of endpoints, such as `UpdateTTL`, `CheckRegister`, `ServiceRegister` and `ServiceDeregister`. Connecting ContainerPilot to Consul can be achieved by running Consul as a client to a cluster (mentioned above). It's easy to run this Consul client agent from ContainerPilot itself. From 189414c009e478c8caccb8c9366db5fba422816f Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Thu, 7 Dec 2017 16:13:16 -0800 Subject: [PATCH 03/32] Resolves #47 by adding CONSUL_RETRY_JOIN_WAN --- README.md | 5 +++++ bin/consul-manage | 5 +++++ local-compose.yml | 29 +++++++++++++++++++++++++++-- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index eb36599..8cf55c4 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,11 @@ In our experience, including a Consul cluster within a project's `docker-compose #### Environment Variables - `CONSUL_DATACENTER_NAME`: sets the name of the data center in which the Consul cluster is running. +- `CONSUL_RETRY_JOIN_WAN`: sets the remote datacenter addresses to join. Must be a valid HCL list (i.e. comma-separated quoted addresses). See Consul's [configuration documentation](https://www.consul.io/docs/agent/options.html#retry_join_wan) for more information. + - The following error will occur if `CONSUL_RETRY_JOIN_WAN` is provided but improperly formatted: + ``` + ==> Error parsing /etc/consul/consul.hcl: ... unexpected token while parsing list: IDENT + ``` ### Clients diff --git a/bin/consul-manage b/bin/consul-manage index b43028b..e23e6c2 100755 --- a/bin/consul-manage +++ b/bin/consul-manage @@ -11,6 +11,11 @@ preStart() { _log "Updating consul datacenter name" sed -i "s/CONSUL_DATACENTER_NAME/${CONSUL_DATACENTER_NAME:-dc1}/" /etc/consul/consul.hcl + + if [ ! -z "$CONSUL_RETRY_JOIN_WAN" ]; then + _log "Updating consul retry_join_wan field: retry_join_wan = [${CONSUL_RETRY_JOIN_WAN}]" + echo "retry_join_wan = [${CONSUL_RETRY_JOIN_WAN}]" >> /etc/consul/consul.hcl + fi } # diff --git a/local-compose.yml b/local-compose.yml index f3c21ba..5fff677 100644 --- a/local-compose.yml +++ b/local-compose.yml @@ -7,14 +7,39 @@ services: # created user-defined network and internal DNS for the name "consul". # Nodes will use Docker DNS for the service (passed in via the CONSUL # env var) to find each other and bootstrap the cluster. - consul: + consul1: build: . + image: autopilotpattern/consul:${TAG:-latest} restart: always mem_limit: 128m ports: - 8500 environment: - - CONSUL=consul + - CONSUL=consul1 - CONSUL_DATACENTER_NAME=dc1 command: > /usr/local/bin/containerpilot + + # The following service block defines a nearly-identical deployment in a + # fictional second datacenter. This can be used to test multi-datacenter + # behavior in Consul. + # NOTE: CONSUL_RETRY_JOIN_WAN *must* be a valid HCL list, + # e.g.: + # "1.1.1.1" + # or + # "1.1.1.1","2.2.2.2" + + consul2: + image: autopilotpattern/consul:${TAG:-latest} + restart: always + mem_limit: 128m + ports: + - 8500 + environment: + - CONSUL=consul2 + - CONSUL_DATACENTER_NAME=dc2 + - CONSUL_RETRY_JOIN_WAN=consul1 + command: > + /usr/local/bin/containerpilot + links: + - consul1 From 23c8c60696a8a693a080001bd6daa29915fd9380 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Thu, 7 Dec 2017 16:24:01 -0800 Subject: [PATCH 04/32] Remove debugging output --- bin/consul-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/consul-manage b/bin/consul-manage index e23e6c2..bb25c71 100755 --- a/bin/consul-manage +++ b/bin/consul-manage @@ -13,7 +13,7 @@ preStart() { sed -i "s/CONSUL_DATACENTER_NAME/${CONSUL_DATACENTER_NAME:-dc1}/" /etc/consul/consul.hcl if [ ! -z "$CONSUL_RETRY_JOIN_WAN" ]; then - _log "Updating consul retry_join_wan field: retry_join_wan = [${CONSUL_RETRY_JOIN_WAN}]" + _log "Updating consul retry_join_wan field" echo "retry_join_wan = [${CONSUL_RETRY_JOIN_WAN}]" >> /etc/consul/consul.hcl fi } From abad90e38bc5e5fac2d55a2516d15585615915f8 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Thu, 7 Dec 2017 16:56:45 -0800 Subject: [PATCH 05/32] Fix the environment value now that we've collected the error message --- local-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local-compose.yml b/local-compose.yml index 5fff677..1664728 100644 --- a/local-compose.yml +++ b/local-compose.yml @@ -38,7 +38,7 @@ services: environment: - CONSUL=consul2 - CONSUL_DATACENTER_NAME=dc2 - - CONSUL_RETRY_JOIN_WAN=consul1 + - CONSUL_RETRY_JOIN_WAN="consul1" command: > /usr/local/bin/containerpilot links: From b15de5282d69bd6107e6afdfd3043a1d6ab88360 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 11 Dec 2017 13:54:03 -0800 Subject: [PATCH 06/32] Environment variable documentation and moving examples. --- README.md | 11 ++++- .../docker-compose-multi-datacenter.yml | 37 +++++++++++++++ examples/compose/docker-compose.yml | 22 +++++++++ .../triton/docker-compose.yml | 0 local-compose.yml | 45 ------------------- 5 files changed, 68 insertions(+), 47 deletions(-) create mode 100644 examples/compose/docker-compose-multi-datacenter.yml create mode 100644 examples/compose/docker-compose.yml rename docker-compose.yml => examples/triton/docker-compose.yml (100%) delete mode 100644 local-compose.yml diff --git a/README.md b/README.md index 8cf55c4..6bbd2fb 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,15 @@ In our experience, including a Consul cluster within a project's `docker-compose #### Environment Variables -- `CONSUL_DATACENTER_NAME`: sets the name of the data center in which the Consul cluster is running. -- `CONSUL_RETRY_JOIN_WAN`: sets the remote datacenter addresses to join. Must be a valid HCL list (i.e. comma-separated quoted addresses). See Consul's [configuration documentation](https://www.consul.io/docs/agent/options.html#retry_join_wan) for more information. +- `CONSUL_DEV`: Enable development mode, allowing a node to self-elect as a cluster leader. Consul flag: [`-dev`](https://www.consul.io/docs/agent/options.html#_dev). + - The following errors will occur if `CONSUL_DEV` is omitted and not enough Consul instances are deployed: + ``` + [ERR] agent: failed to sync remote state: No cluster leader + [ERR] agent: failed to sync changes: No cluster leader + [ERR] agent: Coordinate update error: No cluster leader + ``` +- `CONSUL_DATACENTER_NAME`: Explicitly set the name of the data center in which Consul is running. Consul flag: [`-datacenter`](https://www.consul.io/docs/agent/options.html#datacenter). +- `CONSUL_RETRY_JOIN_WAN`: sets the remote datacenter addresses to join. Must be a valid HCL list (i.e. comma-separated quoted addresses). Consul flag: [`-retry-join-wan`](https://www.consul.io/docs/agent/options.html#retry_join_wan). - The following error will occur if `CONSUL_RETRY_JOIN_WAN` is provided but improperly formatted: ``` ==> Error parsing /etc/consul/consul.hcl: ... unexpected token while parsing list: IDENT diff --git a/examples/compose/docker-compose-multi-datacenter.yml b/examples/compose/docker-compose-multi-datacenter.yml new file mode 100644 index 0000000..537c3fe --- /dev/null +++ b/examples/compose/docker-compose-multi-datacenter.yml @@ -0,0 +1,37 @@ +version: '2.1' + +services: + + # Service definition for "nearby" Consul cluster in a fictional datacenter named "dc1" (the + # default datacenter name). Note the `CONSUL` environment variable tells + # instances of this service how to find each other and the datacenter is explicitly set to "dc1" + consul-dc1: + image: autopilotpattern/consul:${TAG:-latest} + restart: always + mem_limit: 128m + ports: + - 8500 + environment: + - CONSUL=consul-dc1 + - CONSUL_DATACENTER_NAME=dc1 + command: > + /usr/local/bin/containerpilot + + # Service definition for "remotely deployed" Consul cluster named "dc2". + # Note the `CONSUL` environment variable tells instances of this service how to find each other + # while `CONSUL_RETRY_JOIN_WAN` tells them what hostname to use to find instances in the "dc1" + # datacenter. + consul-dc2: + image: autopilotpattern/consul:${TAG:-latest} + restart: always + mem_limit: 128m + ports: + - 8500 + environment: + - CONSUL=consul-dc2 + - CONSUL_DATACENTER_NAME=dc2 + - CONSUL_RETRY_JOIN_WAN="consul-dc1" + command: > + /usr/local/bin/containerpilot + links: + - consul-dc1 \ No newline at end of file diff --git a/examples/compose/docker-compose.yml b/examples/compose/docker-compose.yml new file mode 100644 index 0000000..b90dab9 --- /dev/null +++ b/examples/compose/docker-compose.yml @@ -0,0 +1,22 @@ +version: '2.1' + +services: + + # Service definition for Consul cluster with a minimum of 3 nodes. + # For local development we use Compose v2 so that we have an automatically + # created user-defined network and internal DNS for the name "consul". + # Nodes will use Docker DNS for the service (passed in via the CONSUL + # env var) to find each other and bootstrap the cluster. + # Note: Unless CONSUL_DEV is set, at least three instances are required for quorum. + consul: + build: ../../ + image: autopilotpattern/consul:${TAG:-latest} + restart: always + mem_limit: 128m + ports: + - 8500 + environment: + - CONSUL=consul + - CONSUL_DATACENTER_NAME=dc1 + command: > + /usr/local/bin/containerpilot diff --git a/docker-compose.yml b/examples/triton/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to examples/triton/docker-compose.yml diff --git a/local-compose.yml b/local-compose.yml deleted file mode 100644 index 1664728..0000000 --- a/local-compose.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: '2.1' - -services: - - # Service definition for Consul cluster with a minimum of 3 nodes. - # For local development we use Compose v2 so that we have an automatically - # created user-defined network and internal DNS for the name "consul". - # Nodes will use Docker DNS for the service (passed in via the CONSUL - # env var) to find each other and bootstrap the cluster. - consul1: - build: . - image: autopilotpattern/consul:${TAG:-latest} - restart: always - mem_limit: 128m - ports: - - 8500 - environment: - - CONSUL=consul1 - - CONSUL_DATACENTER_NAME=dc1 - command: > - /usr/local/bin/containerpilot - - # The following service block defines a nearly-identical deployment in a - # fictional second datacenter. This can be used to test multi-datacenter - # behavior in Consul. - # NOTE: CONSUL_RETRY_JOIN_WAN *must* be a valid HCL list, - # e.g.: - # "1.1.1.1" - # or - # "1.1.1.1","2.2.2.2" - - consul2: - image: autopilotpattern/consul:${TAG:-latest} - restart: always - mem_limit: 128m - ports: - - 8500 - environment: - - CONSUL=consul2 - - CONSUL_DATACENTER_NAME=dc2 - - CONSUL_RETRY_JOIN_WAN="consul1" - command: > - /usr/local/bin/containerpilot - links: - - consul1 From ee400d621d1a76c6545b8bcfd705f0436a51afa7 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 11 Dec 2017 14:15:14 -0800 Subject: [PATCH 07/32] Triton datacenter autodetection --- bin/consul-manage | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/bin/consul-manage b/bin/consul-manage index b43028b..8a47749 100755 --- a/bin/consul-manage +++ b/bin/consul-manage @@ -9,8 +9,17 @@ preStart() { _log "Updating consul advertise address" sed -i "s/CONTAINERPILOT_CONSUL_IP/${CONTAINERPILOT_CONSUL_IP}/" /etc/consul/consul.hcl - _log "Updating consul datacenter name" - sed -i "s/CONSUL_DATACENTER_NAME/${CONSUL_DATACENTER_NAME:-dc1}/" /etc/consul/consul.hcl + if [ -n "$CONSUL_DATACENTER_NAME" ]; then + _log "Updating consul datacenter name (specified: '${CONSUL_DATACENTER_NAME}' )" + sed -i "s/CONSUL_DATACENTER_NAME/${CONSUL_DATACENTER_NAME}/" /etc/consul/consul.hcl + elif [ -f "/native/usr/sbin/mdata-get" ]; then + DETECTED_DATACENTER_NAME=$(/native/usr/sbin/mdata-get sdc:datacenter_name) + _log "Updating consul datacenter name (detected: '${DETECTED_DATACENTER_NAME}')" + sed -i "s/CONSUL_DATACENTER_NAME/${DETECTED_DATACENTER_NAME}/" /etc/consul/consul.hcl + else + _log "Updating consul datacenter name (default: 'dc1')" + sed -i "s/CONSUL_DATACENTER_NAME/dc1/" /etc/consul/consul.hcl + fi } # From 5c4c2d4e7581d546e633c3c46af3c7aae9bba183 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 11 Dec 2017 14:46:46 -0800 Subject: [PATCH 08/32] README update for envs --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eb36599..00b2407 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,17 @@ In our experience, including a Consul cluster within a project's `docker-compose #### Environment Variables -- `CONSUL_DATACENTER_NAME`: sets the name of the data center in which the Consul cluster is running. +- `CONSUL_DEV`: Enable development mode, allowing a node to self-elect as a cluster leader. Consul flag: [`-dev`](https://www.consul.io/docs/agent/options.html#_dev). + - The following errors will occur if `CONSUL_DEV` is omitted and not enough Consul instances are deployed: + ``` + [ERR] agent: failed to sync remote state: No cluster leader + [ERR] agent: failed to sync changes: No cluster leader + [ERR] agent: Coordinate update error: No cluster leader + ``` +- `CONSUL_DATACENTER_NAME`: Explicitly set the name of the data center in which Consul is running. Consul flag: [`-datacenter`](https://www.consul.io/docs/agent/options.html#datacenter). + - If this variable is specified it will be used as-is. + - If not specified, automatic detection of the datacenter will be attempted. See [issue #23](https://github.com/autopilotpattern/consul/issues/23) for more details. + - Consul's default of "dc1" will be used if none of the above apply. ### Clients From 32ed8fcf77b818021b1d4047be7e21abc59d6982 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Tue, 12 Dec 2017 18:24:30 -0800 Subject: [PATCH 09/32] more things --- .gitignore | 3 +- README.md | 27 +++++ bin/consul-manage | 19 ++++ ...cker-compose-multi-datacenter.yml.template | 22 ++++ examples/triton/docker-compose.yml | 2 +- makefile | 8 ++ setup-multi-datacenter.sh | 103 ++++++++++++++++++ setup.sh | 16 +-- 8 files changed, 191 insertions(+), 9 deletions(-) create mode 100644 examples/triton/docker-compose-multi-datacenter.yml.template create mode 100755 setup-multi-datacenter.sh diff --git a/.gitignore b/.gitignore index 1ab2a9a..ad24b45 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -_env +_env* +examples/triton/docker-compose-*.yml diff --git a/README.md b/README.md index 18953d3..dc1220f 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,30 @@ $ docker exec -it consul_consul_3 consul info | grep num_peers ``` +### Run it with more than one datacenter! + +Follow the same steps as above until you reach `./setup.sh` but execute `./setup-multi-datacenter.sh` instead, providing as arguments Triton profiles which belong to the desired data centers. + +Since interacting with multiple data centers requires switching between Triton profiles it's easier to perform the following steps in separate terminals. It is possible to perform all the steps for a single data center and then change profiles. Additionally, setting `COMPOSE_PROJECT_NAME` to match the profile or data center will help distinguish nodes in Triton Portal and the `triton instance ls` listing. + +Execute the following commands, once per data center, from the project root: + +``` +$ eval "$(TRITON_PROFILE= triton env -d)" + +$ export COMPOSE_FILE=examples/triton/docker-compose-.yml + +# the following is not strictly necessary but helps to discern between clusters +$ export COMPOSE_PROJECT_NAME= + +$ docker-compose up -d +Creating _consul_1 ... done + +$ docker-compose scale consul=3 +``` + +Note: the `cns.joyent.com` hostnames cannot be resolved from outside the datacenters. Change `cns.joyent.com` to `triton.zone` to access the web UI. + ## Using this in your own composition There are two ways to run Consul and both come into play when deploying ContainerPilot, a cluster of Consul servers and individual Consul client agents. @@ -100,6 +124,9 @@ In our experience, including a Consul cluster within a project's `docker-compose ``` ==> Error parsing /etc/consul/consul.hcl: ... unexpected token while parsing list: IDENT ``` + - Gossip over the WAN requires the following ports to be accessible between data centers, make sure that adequate firewall rules have been established for the following ports by default: + - `8300`: Server RPC port (TCP) + - `8302`: Serf WAN gossip port (TCP + UDP) ### Clients diff --git a/bin/consul-manage b/bin/consul-manage index 1131979..32965ef 100755 --- a/bin/consul-manage +++ b/bin/consul-manage @@ -23,7 +23,26 @@ preStart() { if [ -n "$CONSUL_RETRY_JOIN_WAN" ]; then _log "Updating consul retry_join_wan field" + sed -i '/^retry_join_wan/d' /etc/consul/consul.hcl echo "retry_join_wan = [${CONSUL_RETRY_JOIN_WAN}]" >> /etc/consul/consul.hcl + + IP_ADDRESS=$(hostname -i) + + # the serf_wan_bind field was recently renamed to serf_wan + # if this field is not set connections wan joins will be refused + sed -i '/^serf_wan/d' /etc/consul/consul.hcl + _log "Updating consul serf_wan field" + echo "serf_wan = \"${IP_ADDRESS}\"" >> /etc/consul/consul.hcl + + # advertise_addr_wan allows nodes + sed -i '/^advertise_addr_wan/d' /etc/consul/consul.hcl + _log "Updating consul advertise_addr_wan field" + echo "advertise_addr_wan = \"${IP_ADDRESS}\"" >> /etc/consul/consul.hcl + + # translate_wan_addrs allows us to reach remote nodes through their advertise_addr_wan + sed -i '/^translate_wan_addrs/d' /etc/consul/consul.hcl + _log "Updating consul translate_wan_addrs field" + echo "translate_wan_addrs = true" >> /etc/consul/consul.hcl fi } diff --git a/examples/triton/docker-compose-multi-datacenter.yml.template b/examples/triton/docker-compose-multi-datacenter.yml.template new file mode 100644 index 0000000..23520d8 --- /dev/null +++ b/examples/triton/docker-compose-multi-datacenter.yml.template @@ -0,0 +1,22 @@ +version: '2.1' + +services: + + # Service definition for Consul cluster running in us-east-1. + # Cloned by ../../setup-multi-datacenter.sh once per profile + consul: + image: tjcelaya/consul:56 + labels: + - triton.cns.services=consul + restart: always + mem_limit: 128m + ports: + - 8300 # Server RPC port + - "8302/tcp" # Serf WAN port + - "8302/udp" # Serf WAN port + - 8500 + env_file: + - ENV_FILE_NAME + network_mode: bridge + command: > + /usr/local/bin/containerpilot \ No newline at end of file diff --git a/examples/triton/docker-compose.yml b/examples/triton/docker-compose.yml index d2ab4ec..b395387 100644 --- a/examples/triton/docker-compose.yml +++ b/examples/triton/docker-compose.yml @@ -6,7 +6,7 @@ services: # Nodes will use Triton CNS for the service (passed in via the CONSUL # env var) to find each other and bootstrap the cluster. consul: - image: autopilotpattern/consul:${TAG:-latest} + image: tjcelaya/consul:47 labels: - triton.cns.services=consul restart: always diff --git a/makefile b/makefile index 822b929..84ed493 100644 --- a/makefile +++ b/makefile @@ -81,6 +81,14 @@ test/triton: test/triton/dev: ./test/triton.sh + +# ------------------------------------------------ +# Multi-datacenter usage +mdc-clean: + rm -rf examples/triton/_env* examples/triton/docker-compose-*.yml + + + ## Print environment for build debugging debug: @echo GIT_COMMIT=$(GIT_COMMIT) diff --git a/setup-multi-datacenter.sh b/setup-multi-datacenter.sh new file mode 100755 index 0000000..8dd81ed --- /dev/null +++ b/setup-multi-datacenter.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -e -o pipefail + +help() { + echo + echo 'Usage ./setup-multi-datacenter.sh [ [...]]' + echo + echo 'Invokes ./setup repeatedly to create one _env file per datacenter per triton profile,' + echo 'attempting to preserve the triton profile set before this script was invoked.' + echo + echo 'Warning: The current triton profile will be changed for each invocation of ./setup.sh,' + echo 'this may cause unexpected behavior if other commands that read the current triton profile' + echo 'are executed concurrently!' +} + +if [ "$#" -lt 1 ]; then + help + exit 1 +fi + +declare -a written +declare -a consul_hostnames + +# check that we won't overwrite any _env files first +if [ -f "_env" ]; then + echo "Existing env file found, exiting: _env" +fi + +# check the names of _env files we expect to generate +for profile in "$@" +do + if [ -f "_env-$profile" ]; then + echo "Existing env file found, exiting: _env-$profile" + exit 2 + fi + + if [ -f "examples/triton/_env-$profile" ]; then + echo "Existing env file found, exiting: examples/triton/_env-$profile" + exit 3 + fi + + if [ -f "examples/triton/docker-compose-$profile.yml" ]; then + echo "Existing docker-compose file found, exiting: examples/triton/docker-compose-$profile.yml" + exit 4 + fi +done + +# check that the docker-compose.yml template is in the right place +if [ ! -f "examples/triton/docker-compose-multi-datacenter.yml.template" ]; then + echo "Multi-datacenter docker-compose.yml template is missing!" + exit 5 +fi + +INITIAL_PROFILE=$(triton profile get | awk '/name:/{print $2}') + +# invoke ./setup.sh once per profile +for profile in "$@" +do + echo "Temporarily switching profile: $profile" + triton profile set $profile + eval "$(triton env -d)" + ./setup.sh + + unset CONSUL + source examples/triton/_env + + consul_hostnames+=("\"${CONSUL//cns.joyent.com/triton.zone}\"") + + mv examples/triton/_env "examples/triton/_env-$profile" + + cp examples/triton/docker-compose-multi-datacenter.yml.template \ + "examples/triton/docker-compose-$profile.yml" + + sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "examples/triton/docker-compose-$profile.yml" + + written+=("_env-$profile") +done + + +# finalize _env and prepare docker-compose.yml files +for profile in "$@" +do + # add the CONSUL_RETRY_JOIN_WAN addresses to each _env + echo '# Consul multi-DC bootstrap via Triton CNS' >> examples/triton/_env-$profile + echo "CONSUL_RETRY_JOIN_WAN=$(IFS=,; echo "${consul_hostnames[*]}")" >> examples/triton/_env-$profile + + cp examples/triton/docker-compose-multi-datacenter.yml.template \ + "examples/triton/docker-compose-$profile.yml" + + sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "examples/triton/docker-compose-$profile.yml" + + written+=("_env-$profile") +done + +echo "Wrote: $written" + +echo "Clearing triton env" + +eval "$(triton env -u)" +triton profile set "$INITIAL_PROFILE" + +echo "Restored profile: $INITIAL_PROFILE" +echo "You might want to run eval \"\$(triton env) again" \ No newline at end of file diff --git a/setup.sh b/setup.sh index be7f2db..a196427 100755 --- a/setup.sh +++ b/setup.sh @@ -42,9 +42,11 @@ check() { # make sure Docker client is pointed to the same place as the Triton client local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}') local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') - TRITON_USER=$(triton profile get | awk -F": " '/account:/{print $2}') - TRITON_DC=$(triton profile get | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') + + TRITON_USER=$(triton profile get $TRITON_PROFILE | awk -F": " '/account:/{print $2}') + TRITON_DC=$(triton profile get $TRITON_PROFILE | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}') + if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then echo tput rev # reverse @@ -71,12 +73,12 @@ check() { fi # setup environment file - if [ ! -f "_env" ]; then - echo '# Consul bootstrap via Triton CNS' >> _env - echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> _env - echo >> _env + if [ ! -f "examples/triton/_env" ]; then + echo '# Consul bootstrap via Triton CNS' >> examples/triton/_env + echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> examples/triton/_env + echo >> examples/triton/_env else - echo 'Existing _env file found, exiting' + echo 'Existing _env file found at examples/triton/_env, exiting' exit fi } From 5bec8ffe84d23e4fa9bf6b17c1a9dc5e208fdb05 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Wed, 13 Dec 2017 14:25:57 -0800 Subject: [PATCH 10/32] it was bind_addr all along --- bin/consul-manage | 50 +++++++++++++------ etc/consul.hcl | 2 +- ...cker-compose-multi-datacenter.yml.template | 2 +- examples/triton/docker-compose.yml | 2 +- makefile | 3 +- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/bin/consul-manage b/bin/consul-manage index 32965ef..ca430d2 100755 --- a/bin/consul-manage +++ b/bin/consul-manage @@ -1,13 +1,23 @@ #!/bin/bash set -eo pipefail +updateConfigFromEnvOrDefault() { + + _log "Updating consul $1 field" + sed -i "/^$1/d" /etc/consul/consul.hcl + + if [ -n "${!2}" ]; then + echo "$1 = \"${!2}\"" >> /etc/consul/consul.hcl + else + echo "$1 = \"$3\"" >> /etc/consul/consul.hcl + fi +} + # # Update the -advertise address based on the interface that ContainerPilot has # been told to listen on. # preStart() { - _log "Updating consul advertise address" - sed -i "s/CONTAINERPILOT_CONSUL_IP/${CONTAINERPILOT_CONSUL_IP}/" /etc/consul/consul.hcl if [ -n "$CONSUL_DATACENTER_NAME" ]; then _log "Updating consul datacenter name (specified: '${CONSUL_DATACENTER_NAME}' )" @@ -26,23 +36,33 @@ preStart() { sed -i '/^retry_join_wan/d' /etc/consul/consul.hcl echo "retry_join_wan = [${CONSUL_RETRY_JOIN_WAN}]" >> /etc/consul/consul.hcl - IP_ADDRESS=$(hostname -i) - - # the serf_wan_bind field was recently renamed to serf_wan - # if this field is not set connections wan joins will be refused - sed -i '/^serf_wan/d' /etc/consul/consul.hcl - _log "Updating consul serf_wan field" - echo "serf_wan = \"${IP_ADDRESS}\"" >> /etc/consul/consul.hcl - - # advertise_addr_wan allows nodes - sed -i '/^advertise_addr_wan/d' /etc/consul/consul.hcl - _log "Updating consul advertise_addr_wan field" - echo "advertise_addr_wan = \"${IP_ADDRESS}\"" >> /etc/consul/consul.hcl - # translate_wan_addrs allows us to reach remote nodes through their advertise_addr_wan sed -i '/^translate_wan_addrs/d' /etc/consul/consul.hcl _log "Updating consul translate_wan_addrs field" echo "translate_wan_addrs = true" >> /etc/consul/consul.hcl + + IP_ADDRESS=$(hostname -i) + + # TODO: the following envs _can_ be set independently or CONSUL_RETRY_JOIN_WAN so they might + # need to be moved out of this if clause + + # the serf_lan_bind field was recently renamed to serf_wan + # serf_lan tells nodes their address within the LAN + updateConfigFromEnvOrDefault 'serf_lan' 'CONSUL_SERF_LAN_BIND' "$CONTAINERPILOT_CONSUL_IP" + + # advertise_addr tells nodes their private, routeable address + updateConfigFromEnvOrDefault 'advertise_addr' 'CONSUL_ADVERTISE_ADDR' "$CONTAINERPILOT_CONSUL_IP" + + # the serf_wan_bind field was recently renamed to serf_wan + # if this field is not set WAN joins will be refused since the bind address will differ + # from the address used to reach the node + updateConfigFromEnvOrDefault 'serf_wan' 'CONSUL_SERF_WAN_BIND' "$IP_ADDRESS" + + # advertise_addr_wan tells nodes their public address for WAN communication + updateConfigFromEnvOrDefault 'advertise_addr_wan' 'CONSUL_ADVERTISE_ADDR_WAN' "$IP_ADDRESS" + else + _log "Updating consul bind address" + updateConfigFromEnvOrDefault 'bind_addr' 'CONSUL_ADVERTISE_ADDR_WAN' "$CONTAINERPILOT_CONSUL_IP" fi } diff --git a/etc/consul.hcl b/etc/consul.hcl index c8a0330..1c79748 100644 --- a/etc/consul.hcl +++ b/etc/consul.hcl @@ -1,4 +1,4 @@ -bind_addr = "CONTAINERPILOT_CONSUL_IP" +bind_addr = "0.0.0.0" datacenter = "CONSUL_DATACENTER_NAME" data_dir = "/data" client_addr = "0.0.0.0" diff --git a/examples/triton/docker-compose-multi-datacenter.yml.template b/examples/triton/docker-compose-multi-datacenter.yml.template index 23520d8..9001759 100644 --- a/examples/triton/docker-compose-multi-datacenter.yml.template +++ b/examples/triton/docker-compose-multi-datacenter.yml.template @@ -5,7 +5,7 @@ services: # Service definition for Consul cluster running in us-east-1. # Cloned by ../../setup-multi-datacenter.sh once per profile consul: - image: tjcelaya/consul:56 + image: autopilotpattern/consul:${TAG:-latest} labels: - triton.cns.services=consul restart: always diff --git a/examples/triton/docker-compose.yml b/examples/triton/docker-compose.yml index b395387..d2ab4ec 100644 --- a/examples/triton/docker-compose.yml +++ b/examples/triton/docker-compose.yml @@ -6,7 +6,7 @@ services: # Nodes will use Triton CNS for the service (passed in via the CONSUL # env var) to find each other and bootstrap the cluster. consul: - image: tjcelaya/consul:47 + image: autopilotpattern/consul:${TAG:-latest} labels: - triton.cns.services=consul restart: always diff --git a/makefile b/makefile index 84ed493..5516b35 100644 --- a/makefile +++ b/makefile @@ -84,11 +84,10 @@ test/triton/dev: # ------------------------------------------------ # Multi-datacenter usage -mdc-clean: +clean/triton: rm -rf examples/triton/_env* examples/triton/docker-compose-*.yml - ## Print environment for build debugging debug: @echo GIT_COMMIT=$(GIT_COMMIT) From 25de3ddd39d7cf6400bc7ce5eebd17466aa419fe Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Wed, 13 Dec 2017 15:54:14 -0800 Subject: [PATCH 11/32] Always set the alternate addresses for now, might move all of these back into the CONSUL_RETRY_JOIN_WAN block --- README.md | 9 +++++- bin/consul-manage | 78 +++++++++++++++++++++++++++++------------------ 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index dc1220f..8f0980e 100644 --- a/README.md +++ b/README.md @@ -119,12 +119,19 @@ In our experience, including a Consul cluster within a project's `docker-compose - If this variable is specified it will be used as-is. - If not specified, automatic detection of the datacenter will be attempted. See [issue #23](https://github.com/autopilotpattern/consul/issues/23) for more details. - Consul's default of "dc1" will be used if none of the above apply. + +- `CONSUL_BIND_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to `0.0.0.0` if not specified and `CONSUL_RETRY_JOIN_WAN` is provided. Be aware of +- `CONSUL_SERF_LAN_BIND`: Explicitly set the corresponding Consul configuration. This value will be set to the server's private address automatically if not specified. Consul flag: [`-serf-lan-bind`](https://www.consul.io/docs/agent/options.html#serf_lan_bind). +- `CONSUL_SERF_WAN_BIND`: Explicitly set the corresponding Consul configuration. This value will be set to the server's public address automatically if not specified. Consul flag: [`-serf-wan-bind`](https://www.consul.io/docs/agent/options.html#serf_wan_bind). +- `CONSUL_ADVERTISE_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to the server's private address automatically if not specified. Consul flag: [`-advertise-addr`](https://www.consul.io/docs/agent/options.html#advertise_addr). +- `CONSUL_ADVERTISE_ADDR_WAN`: Explicitly set the corresponding Consul configuration. This value will be set to the server's public address automatically if not specified. Consul flag: [`-advertise-addr-wan`](https://www.consul.io/docs/agent/options.html#advertise_addr_wan). + - `CONSUL_RETRY_JOIN_WAN`: sets the remote datacenter addresses to join. Must be a valid HCL list (i.e. comma-separated quoted addresses). Consul flag: [`-retry-join-wan`](https://www.consul.io/docs/agent/options.html#retry_join_wan). - The following error will occur if `CONSUL_RETRY_JOIN_WAN` is provided but improperly formatted: ``` ==> Error parsing /etc/consul/consul.hcl: ... unexpected token while parsing list: IDENT ``` - - Gossip over the WAN requires the following ports to be accessible between data centers, make sure that adequate firewall rules have been established for the following ports by default: + - Gossip over the WAN requires the following ports to be accessible between data centers, make sure that adequate firewall rules have been established for the following ports (this should happen automatically when using docker-compose with Triton): - `8300`: Server RPC port (TCP) - `8302`: Serf WAN gossip port (TCP + UDP) diff --git a/bin/consul-manage b/bin/consul-manage index ca430d2..9c424d8 100755 --- a/bin/consul-manage +++ b/bin/consul-manage @@ -1,18 +1,6 @@ #!/bin/bash set -eo pipefail -updateConfigFromEnvOrDefault() { - - _log "Updating consul $1 field" - sed -i "/^$1/d" /etc/consul/consul.hcl - - if [ -n "${!2}" ]; then - echo "$1 = \"${!2}\"" >> /etc/consul/consul.hcl - else - echo "$1 = \"$3\"" >> /etc/consul/consul.hcl - fi -} - # # Update the -advertise address based on the interface that ContainerPilot has # been told to listen on. @@ -31,6 +19,7 @@ preStart() { sed -i "s/CONSUL_DATACENTER_NAME/dc1/" /etc/consul/consul.hcl fi + if [ -n "$CONSUL_RETRY_JOIN_WAN" ]; then _log "Updating consul retry_join_wan field" sed -i '/^retry_join_wan/d' /etc/consul/consul.hcl @@ -41,29 +30,36 @@ preStart() { _log "Updating consul translate_wan_addrs field" echo "translate_wan_addrs = true" >> /etc/consul/consul.hcl - IP_ADDRESS=$(hostname -i) + # only set bind_addr = 0.0.0.0 if none was specified explicitly with CONSUL_BIND_ADDR + if [ -z "CONSUL_BIND_ADDR" ]; then + sed -i '/^bind_addr/d' /etc/consul/consul.hcl + _log "Updating consul field bind_addr to 0.0.0.0 CONSUL_BIND_ADDR was empty and CONSUL_RETRY_JOIN_WAN was not empty" + echo "bind_addr = true" >> /etc/consul/consul.hcl + else + updateConfigFromEnvOrDefault 'bind_addr' 'CONSUL_BIND_ADDR' "$CONTAINERPILOT_CONSUL_IP" + fi + else + # if no WAN addresses were provided, set the bind_addr to the private address + updateConfigFromEnvOrDefault 'bind_addr' 'CONSUL_BIND_ADDR' "$CONTAINERPILOT_CONSUL_IP" + fi - # TODO: the following envs _can_ be set independently or CONSUL_RETRY_JOIN_WAN so they might - # need to be moved out of this if clause - # the serf_lan_bind field was recently renamed to serf_wan - # serf_lan tells nodes their address within the LAN - updateConfigFromEnvOrDefault 'serf_lan' 'CONSUL_SERF_LAN_BIND' "$CONTAINERPILOT_CONSUL_IP" + IP_ADDRESS=$(hostname -i) - # advertise_addr tells nodes their private, routeable address - updateConfigFromEnvOrDefault 'advertise_addr' 'CONSUL_ADVERTISE_ADDR' "$CONTAINERPILOT_CONSUL_IP" + # the serf_lan_bind field was recently renamed to serf_wan + # serf_lan tells nodes their address within the LAN + updateConfigFromEnvOrDefault 'serf_lan' 'CONSUL_SERF_LAN_BIND' "$CONTAINERPILOT_CONSUL_IP" - # the serf_wan_bind field was recently renamed to serf_wan - # if this field is not set WAN joins will be refused since the bind address will differ - # from the address used to reach the node - updateConfigFromEnvOrDefault 'serf_wan' 'CONSUL_SERF_WAN_BIND' "$IP_ADDRESS" + # the serf_wan_bind field was recently renamed to serf_wan + # if this field is not set WAN joins will be refused since the bind address will differ + # from the address used to reach the node + updateConfigFromEnvOrDefault 'serf_wan' 'CONSUL_SERF_WAN_BIND' "$IP_ADDRESS" - # advertise_addr_wan tells nodes their public address for WAN communication - updateConfigFromEnvOrDefault 'advertise_addr_wan' 'CONSUL_ADVERTISE_ADDR_WAN' "$IP_ADDRESS" - else - _log "Updating consul bind address" - updateConfigFromEnvOrDefault 'bind_addr' 'CONSUL_ADVERTISE_ADDR_WAN' "$CONTAINERPILOT_CONSUL_IP" - fi + # advertise_addr tells nodes their private, routeable address + updateConfigFromEnvOrDefault 'advertise_addr' 'CONSUL_ADVERTISE_ADDR' "$CONTAINERPILOT_CONSUL_IP" + + # advertise_addr_wan tells nodes their public address for WAN communication + updateConfigFromEnvOrDefault 'advertise_addr_wan' 'CONSUL_ADVERTISE_ADDR_WAN' "$IP_ADDRESS" } # @@ -88,6 +84,28 @@ _log() { echo " $(date -u '+%Y-%m-%d %H:%M:%S') containerpilot: $@" } + +# +# Defines $1 in the consul configuration as either an env or a default. +# This basically behaves like ${!name_of_var} and ${var:-default} together +# but separates the indirect reference from the default so it's more obvious +# +# Check if $2 is the name of a defined environment variable and use ${!2} to +# reference it indirectly. +# +# If it is not defined, use $3 as the value +# +updateConfigFromEnvOrDefault() { + _log "Updating consul field $1" + sed -i "/^$1/d" /etc/consul/consul.hcl + + if [ -n "${!2}" ]; then + echo "$1 = \"${!2}\"" >> /etc/consul/consul.hcl + else + echo "$1 = \"$3\"" >> /etc/consul/consul.hcl + fi +} + # --------------------------------------------------- # parse arguments From 0c3db5815c334612a940c449aa53fe56baafa01f Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Wed, 13 Dec 2017 16:04:10 -0800 Subject: [PATCH 12/32] Fix the CONSUL_BIND_ADDR check --- bin/consul-manage | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/consul-manage b/bin/consul-manage index 9c424d8..e30e298 100755 --- a/bin/consul-manage +++ b/bin/consul-manage @@ -31,12 +31,12 @@ preStart() { echo "translate_wan_addrs = true" >> /etc/consul/consul.hcl # only set bind_addr = 0.0.0.0 if none was specified explicitly with CONSUL_BIND_ADDR - if [ -z "CONSUL_BIND_ADDR" ]; then + if [ -n "$CONSUL_BIND_ADDR" ]; then + updateConfigFromEnvOrDefault 'bind_addr' 'CONSUL_BIND_ADDR' "$CONTAINERPILOT_CONSUL_IP" + else sed -i '/^bind_addr/d' /etc/consul/consul.hcl _log "Updating consul field bind_addr to 0.0.0.0 CONSUL_BIND_ADDR was empty and CONSUL_RETRY_JOIN_WAN was not empty" echo "bind_addr = true" >> /etc/consul/consul.hcl - else - updateConfigFromEnvOrDefault 'bind_addr' 'CONSUL_BIND_ADDR' "$CONTAINERPILOT_CONSUL_IP" fi else # if no WAN addresses were provided, set the bind_addr to the private address From b44294b9c78c0631e519994bf2fd8ddf09085288 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Thu, 14 Dec 2017 15:36:02 -0800 Subject: [PATCH 13/32] Add self-anti-affinity --- examples/triton/docker-compose-multi-datacenter.yml.template | 1 + examples/triton/docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/triton/docker-compose-multi-datacenter.yml.template b/examples/triton/docker-compose-multi-datacenter.yml.template index 9001759..a45af29 100644 --- a/examples/triton/docker-compose-multi-datacenter.yml.template +++ b/examples/triton/docker-compose-multi-datacenter.yml.template @@ -8,6 +8,7 @@ services: image: autopilotpattern/consul:${TAG:-latest} labels: - triton.cns.services=consul + - com.docker.swarm.affinities=["container!=~*consul*"] restart: always mem_limit: 128m ports: diff --git a/examples/triton/docker-compose.yml b/examples/triton/docker-compose.yml index d2ab4ec..7cdcd1f 100644 --- a/examples/triton/docker-compose.yml +++ b/examples/triton/docker-compose.yml @@ -9,6 +9,7 @@ services: image: autopilotpattern/consul:${TAG:-latest} labels: - triton.cns.services=consul + - com.docker.swarm.affinities=["container!=~*consul*"] restart: always mem_limit: 128m ports: From 3ebb13807654c0ba5b95943dd6a729006924796a Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Thu, 14 Dec 2017 16:26:50 -0800 Subject: [PATCH 14/32] Fix silly config error --- bin/consul-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/consul-manage b/bin/consul-manage index e30e298..33257d5 100755 --- a/bin/consul-manage +++ b/bin/consul-manage @@ -36,7 +36,7 @@ preStart() { else sed -i '/^bind_addr/d' /etc/consul/consul.hcl _log "Updating consul field bind_addr to 0.0.0.0 CONSUL_BIND_ADDR was empty and CONSUL_RETRY_JOIN_WAN was not empty" - echo "bind_addr = true" >> /etc/consul/consul.hcl + echo "bind_addr = \"0.0.0.0\"" >> /etc/consul/consul.hcl fi else # if no WAN addresses were provided, set the bind_addr to the private address From 24ae977bcfd65666ee1c07c05a4689885dca7586 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Fri, 15 Dec 2017 15:40:21 -0800 Subject: [PATCH 15/32] Move env vars section --- README.md | 58 +++++++++++++++++++++++------------------------ bin/consul-manage | 1 - 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 8f0980e..26ce60c 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,35 @@ $ docker-compose scale consul=3 Note: the `cns.joyent.com` hostnames cannot be resolved from outside the datacenters. Change `cns.joyent.com` to `triton.zone` to access the web UI. +## Environment Variables + +- `CONSUL_DEV`: Enable development mode, allowing a node to self-elect as a cluster leader. Consul flag: [`-dev`](https://www.consul.io/docs/agent/options.html#_dev). + - The following errors will occur if `CONSUL_DEV` is omitted and not enough Consul instances are deployed: + ``` + [ERR] agent: failed to sync remote state: No cluster leader + [ERR] agent: failed to sync changes: No cluster leader + [ERR] agent: Coordinate update error: No cluster leader + ``` +- `CONSUL_DATACENTER_NAME`: Explicitly set the name of the data center in which Consul is running. Consul flag: [`-datacenter`](https://www.consul.io/docs/agent/options.html#datacenter). + - If this variable is specified it will be used as-is. + - If not specified, automatic detection of the datacenter will be attempted. See [issue #23](https://github.com/autopilotpattern/consul/issues/23) for more details. + - Consul's default of "dc1" will be used if none of the above apply. + +- `CONSUL_BIND_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to `0.0.0.0` if not specified and `CONSUL_RETRY_JOIN_WAN` is provided. Be aware of +- `CONSUL_SERF_LAN_BIND`: Explicitly set the corresponding Consul configuration. This value will be set to the server's private address automatically if not specified. Consul flag: [`-serf-lan-bind`](https://www.consul.io/docs/agent/options.html#serf_lan_bind). +- `CONSUL_SERF_WAN_BIND`: Explicitly set the corresponding Consul configuration. This value will be set to the server's public address automatically if not specified. Consul flag: [`-serf-wan-bind`](https://www.consul.io/docs/agent/options.html#serf_wan_bind). +- `CONSUL_ADVERTISE_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to the server's private address automatically if not specified. Consul flag: [`-advertise-addr`](https://www.consul.io/docs/agent/options.html#advertise_addr). +- `CONSUL_ADVERTISE_ADDR_WAN`: Explicitly set the corresponding Consul configuration. This value will be set to the server's public address automatically if not specified. Consul flag: [`-advertise-addr-wan`](https://www.consul.io/docs/agent/options.html#advertise_addr_wan). + +- `CONSUL_RETRY_JOIN_WAN`: sets the remote datacenter addresses to join. Must be a valid HCL list (i.e. comma-separated quoted addresses). Consul flag: [`-retry-join-wan`](https://www.consul.io/docs/agent/options.html#retry_join_wan). + - The following error will occur if `CONSUL_RETRY_JOIN_WAN` is provided but improperly formatted: + ``` + ==> Error parsing /etc/consul/consul.hcl: ... unexpected token while parsing list: IDENT + ``` + - Gossip over the WAN requires the following ports to be accessible between data centers, make sure that adequate firewall rules have been established for the following ports (this should happen automatically when using docker-compose with Triton): + - `8300`: Server RPC port (TCP) + - `8302`: Serf WAN gossip port (TCP + UDP) + ## Using this in your own composition There are two ways to run Consul and both come into play when deploying ContainerPilot, a cluster of Consul servers and individual Consul client agents. @@ -106,35 +135,6 @@ services: In our experience, including a Consul cluster within a project's `docker-compose.yml` can help developers understand and test how a service should be discovered and registered within a wider infrastructure context. -#### Environment Variables - -- `CONSUL_DEV`: Enable development mode, allowing a node to self-elect as a cluster leader. Consul flag: [`-dev`](https://www.consul.io/docs/agent/options.html#_dev). - - The following errors will occur if `CONSUL_DEV` is omitted and not enough Consul instances are deployed: - ``` - [ERR] agent: failed to sync remote state: No cluster leader - [ERR] agent: failed to sync changes: No cluster leader - [ERR] agent: Coordinate update error: No cluster leader - ``` -- `CONSUL_DATACENTER_NAME`: Explicitly set the name of the data center in which Consul is running. Consul flag: [`-datacenter`](https://www.consul.io/docs/agent/options.html#datacenter). - - If this variable is specified it will be used as-is. - - If not specified, automatic detection of the datacenter will be attempted. See [issue #23](https://github.com/autopilotpattern/consul/issues/23) for more details. - - Consul's default of "dc1" will be used if none of the above apply. - -- `CONSUL_BIND_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to `0.0.0.0` if not specified and `CONSUL_RETRY_JOIN_WAN` is provided. Be aware of -- `CONSUL_SERF_LAN_BIND`: Explicitly set the corresponding Consul configuration. This value will be set to the server's private address automatically if not specified. Consul flag: [`-serf-lan-bind`](https://www.consul.io/docs/agent/options.html#serf_lan_bind). -- `CONSUL_SERF_WAN_BIND`: Explicitly set the corresponding Consul configuration. This value will be set to the server's public address automatically if not specified. Consul flag: [`-serf-wan-bind`](https://www.consul.io/docs/agent/options.html#serf_wan_bind). -- `CONSUL_ADVERTISE_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to the server's private address automatically if not specified. Consul flag: [`-advertise-addr`](https://www.consul.io/docs/agent/options.html#advertise_addr). -- `CONSUL_ADVERTISE_ADDR_WAN`: Explicitly set the corresponding Consul configuration. This value will be set to the server's public address automatically if not specified. Consul flag: [`-advertise-addr-wan`](https://www.consul.io/docs/agent/options.html#advertise_addr_wan). - -- `CONSUL_RETRY_JOIN_WAN`: sets the remote datacenter addresses to join. Must be a valid HCL list (i.e. comma-separated quoted addresses). Consul flag: [`-retry-join-wan`](https://www.consul.io/docs/agent/options.html#retry_join_wan). - - The following error will occur if `CONSUL_RETRY_JOIN_WAN` is provided but improperly formatted: - ``` - ==> Error parsing /etc/consul/consul.hcl: ... unexpected token while parsing list: IDENT - ``` - - Gossip over the WAN requires the following ports to be accessible between data centers, make sure that adequate firewall rules have been established for the following ports (this should happen automatically when using docker-compose with Triton): - - `8300`: Server RPC port (TCP) - - `8302`: Serf WAN gossip port (TCP + UDP) - ### Clients ContainerPilot utilizes Consul's [HTTP Agent API](https://www.consul.io/api/agent.html) for a handful of endpoints, such as `UpdateTTL`, `CheckRegister`, `ServiceRegister` and `ServiceDeregister`. Connecting ContainerPilot to Consul can be achieved by running Consul as a client to a cluster (mentioned above). It's easy to run this Consul client agent from ContainerPilot itself. diff --git a/bin/consul-manage b/bin/consul-manage index 33257d5..04bada0 100755 --- a/bin/consul-manage +++ b/bin/consul-manage @@ -43,7 +43,6 @@ preStart() { updateConfigFromEnvOrDefault 'bind_addr' 'CONSUL_BIND_ADDR' "$CONTAINERPILOT_CONSUL_IP" fi - IP_ADDRESS=$(hostname -i) # the serf_lan_bind field was recently renamed to serf_wan From ac0d6e2459f929b44c37854df128d49ede4b33b5 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 18 Dec 2017 16:01:21 -0800 Subject: [PATCH 16/32] Split up triton from multi-dc triton examples --- README.md | 2 +- .../docker-compose-multi-dc.yml.template} | 0 .../multi-dc/setup-multi-datacenter.sh | 30 +++++++++---------- setup.sh => examples/triton/setup.sh | 0 4 files changed, 16 insertions(+), 16 deletions(-) rename examples/{triton/docker-compose-multi-datacenter.yml.template => multi-dc/docker-compose-multi-dc.yml.template} (100%) rename setup-multi-datacenter.sh => examples/multi-dc/setup-multi-datacenter.sh (66%) rename setup.sh => examples/triton/setup.sh (100%) diff --git a/README.md b/README.md index b5ca862..515568d 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Note: the `cns.joyent.com` hostnames cannot be resolved from outside the datacen - If not specified, automatic detection of the datacenter will be attempted. See [issue #23](https://github.com/autopilotpattern/consul/issues/23) for more details. - Consul's default of "dc1" will be used if none of the above apply. -- `CONSUL_BIND_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to `0.0.0.0` if not specified and `CONSUL_RETRY_JOIN_WAN` is provided. Be aware of +- `CONSUL_BIND_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to `0.0.0.0` if `CONSUL_BIND_ADDR` is not specified and `CONSUL_RETRY_JOIN_WAN` is provided. Be aware of the security implications of binding the server to a public address and consider setting up encryption or using a VPN to isolate WAN traffic from the public internet. - `CONSUL_SERF_LAN_BIND`: Explicitly set the corresponding Consul configuration. This value will be set to the server's private address automatically if not specified. Consul flag: [`-serf-lan-bind`](https://www.consul.io/docs/agent/options.html#serf_lan_bind). - `CONSUL_SERF_WAN_BIND`: Explicitly set the corresponding Consul configuration. This value will be set to the server's public address automatically if not specified. Consul flag: [`-serf-wan-bind`](https://www.consul.io/docs/agent/options.html#serf_wan_bind). - `CONSUL_ADVERTISE_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to the server's private address automatically if not specified. Consul flag: [`-advertise-addr`](https://www.consul.io/docs/agent/options.html#advertise_addr). diff --git a/examples/triton/docker-compose-multi-datacenter.yml.template b/examples/multi-dc/docker-compose-multi-dc.yml.template similarity index 100% rename from examples/triton/docker-compose-multi-datacenter.yml.template rename to examples/multi-dc/docker-compose-multi-dc.yml.template diff --git a/setup-multi-datacenter.sh b/examples/multi-dc/setup-multi-datacenter.sh similarity index 66% rename from setup-multi-datacenter.sh rename to examples/multi-dc/setup-multi-datacenter.sh index 8dd81ed..d02ec90 100755 --- a/setup-multi-datacenter.sh +++ b/examples/multi-dc/setup-multi-datacenter.sh @@ -34,19 +34,19 @@ do exit 2 fi - if [ -f "examples/triton/_env-$profile" ]; then - echo "Existing env file found, exiting: examples/triton/_env-$profile" + if [ -f "_env-$profile" ]; then + echo "Existing env file found, exiting: _env-$profile" exit 3 fi - if [ -f "examples/triton/docker-compose-$profile.yml" ]; then - echo "Existing docker-compose file found, exiting: examples/triton/docker-compose-$profile.yml" + if [ -f "docker-compose-$profile.yml" ]; then + echo "Existing docker-compose file found, exiting: docker-compose-$profile.yml" exit 4 fi done # check that the docker-compose.yml template is in the right place -if [ ! -f "examples/triton/docker-compose-multi-datacenter.yml.template" ]; then +if [ ! -f "docker-compose-multi-datacenter.yml.template" ]; then echo "Multi-datacenter docker-compose.yml template is missing!" exit 5 fi @@ -62,16 +62,16 @@ do ./setup.sh unset CONSUL - source examples/triton/_env + source _env consul_hostnames+=("\"${CONSUL//cns.joyent.com/triton.zone}\"") - mv examples/triton/_env "examples/triton/_env-$profile" + mv _env "_env-$profile" - cp examples/triton/docker-compose-multi-datacenter.yml.template \ - "examples/triton/docker-compose-$profile.yml" + cp docker-compose-multi-datacenter.yml.template \ + "docker-compose-$profile.yml" - sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "examples/triton/docker-compose-$profile.yml" + sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "docker-compose-$profile.yml" written+=("_env-$profile") done @@ -81,13 +81,13 @@ done for profile in "$@" do # add the CONSUL_RETRY_JOIN_WAN addresses to each _env - echo '# Consul multi-DC bootstrap via Triton CNS' >> examples/triton/_env-$profile - echo "CONSUL_RETRY_JOIN_WAN=$(IFS=,; echo "${consul_hostnames[*]}")" >> examples/triton/_env-$profile + echo '# Consul multi-DC bootstrap via Triton CNS' >> _env-$profile + echo "CONSUL_RETRY_JOIN_WAN=$(IFS=,; echo "${consul_hostnames[*]}")" >> _env-$profile - cp examples/triton/docker-compose-multi-datacenter.yml.template \ - "examples/triton/docker-compose-$profile.yml" + cp docker-compose-multi-datacenter.yml.template \ + "docker-compose-$profile.yml" - sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "examples/triton/docker-compose-$profile.yml" + sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "docker-compose-$profile.yml" written+=("_env-$profile") done diff --git a/setup.sh b/examples/triton/setup.sh similarity index 100% rename from setup.sh rename to examples/triton/setup.sh From a84a860269d641235ded67705f0afc3881f7a49b Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 18 Dec 2017 16:01:55 -0800 Subject: [PATCH 17/32] Remove fake multi-dc docker-compose example --- .../docker-compose-multi-datacenter.yml | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100644 examples/compose/docker-compose-multi-datacenter.yml diff --git a/examples/compose/docker-compose-multi-datacenter.yml b/examples/compose/docker-compose-multi-datacenter.yml deleted file mode 100644 index 537c3fe..0000000 --- a/examples/compose/docker-compose-multi-datacenter.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: '2.1' - -services: - - # Service definition for "nearby" Consul cluster in a fictional datacenter named "dc1" (the - # default datacenter name). Note the `CONSUL` environment variable tells - # instances of this service how to find each other and the datacenter is explicitly set to "dc1" - consul-dc1: - image: autopilotpattern/consul:${TAG:-latest} - restart: always - mem_limit: 128m - ports: - - 8500 - environment: - - CONSUL=consul-dc1 - - CONSUL_DATACENTER_NAME=dc1 - command: > - /usr/local/bin/containerpilot - - # Service definition for "remotely deployed" Consul cluster named "dc2". - # Note the `CONSUL` environment variable tells instances of this service how to find each other - # while `CONSUL_RETRY_JOIN_WAN` tells them what hostname to use to find instances in the "dc1" - # datacenter. - consul-dc2: - image: autopilotpattern/consul:${TAG:-latest} - restart: always - mem_limit: 128m - ports: - - 8500 - environment: - - CONSUL=consul-dc2 - - CONSUL_DATACENTER_NAME=dc2 - - CONSUL_RETRY_JOIN_WAN="consul-dc1" - command: > - /usr/local/bin/containerpilot - links: - - consul-dc1 \ No newline at end of file From 667f0e226208fe412fc45031a42693e31172c521 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 18 Dec 2017 16:17:05 -0800 Subject: [PATCH 18/32] Fix test Dockerfile paths --- test/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Dockerfile b/test/Dockerfile index db9cd52..5740ac9 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -22,10 +22,10 @@ RUN sed -i 's/1.9.0/1.10.0/' /usr/local/bin/triton-docker \ # install test targets -COPY local-compose.yml /src/local-compose.yml -COPY docker-compose.yml /src/docker-compose.yml +COPY examples/compose/docker-compose.yml /src/local-compose.yml +COPY examples/triton/docker-compose.yml /src/docker-compose.yml # install test code COPY test/triton.sh /src/triton.sh COPY test/compose.sh /src/compose.sh -COPY setup.sh /src/setup.sh +COPY examples/triton/setup.sh /src/setup.sh From 5ab7825e06829c68f56bb94587f85158eabd7ed0 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 18 Dec 2017 16:38:33 -0800 Subject: [PATCH 19/32] Fix multi-dc clean target --- makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefile b/makefile index 5516b35..b8b64e8 100644 --- a/makefile +++ b/makefile @@ -84,8 +84,8 @@ test/triton/dev: # ------------------------------------------------ # Multi-datacenter usage -clean/triton: - rm -rf examples/triton/_env* examples/triton/docker-compose-*.yml +clean/multi-dc: + rm -rf examples/multi-dc/_env* examples/multi-dc/docker-compose-*.yml ## Print environment for build debugging From efb1e9b519f9b0d3cfe06b8afffad4d8f44eafde Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 18 Dec 2017 16:40:03 -0800 Subject: [PATCH 20/32] Fix template name --- examples/multi-dc/setup-multi-datacenter.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/multi-dc/setup-multi-datacenter.sh b/examples/multi-dc/setup-multi-datacenter.sh index d02ec90..2c8aa14 100755 --- a/examples/multi-dc/setup-multi-datacenter.sh +++ b/examples/multi-dc/setup-multi-datacenter.sh @@ -46,7 +46,7 @@ do done # check that the docker-compose.yml template is in the right place -if [ ! -f "docker-compose-multi-datacenter.yml.template" ]; then +if [ ! -f "docker-compose-multi-dc.yml.template" ]; then echo "Multi-datacenter docker-compose.yml template is missing!" exit 5 fi @@ -68,7 +68,7 @@ do mv _env "_env-$profile" - cp docker-compose-multi-datacenter.yml.template \ + cp docker-compose-multi-dc.yml.template \ "docker-compose-$profile.yml" sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "docker-compose-$profile.yml" @@ -84,7 +84,7 @@ do echo '# Consul multi-DC bootstrap via Triton CNS' >> _env-$profile echo "CONSUL_RETRY_JOIN_WAN=$(IFS=,; echo "${consul_hostnames[*]}")" >> _env-$profile - cp docker-compose-multi-datacenter.yml.template \ + cp docker-compose-multi-dc.yml.template \ "docker-compose-$profile.yml" sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "docker-compose-$profile.yml" From ff95e50c3c398803041329ae9a6ef2725e539fae Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 18 Dec 2017 16:52:39 -0800 Subject: [PATCH 21/32] Try a new multi-dc layout --- .../docker-compose-multi-dc.yml.template | 0 .../setup-multi-dc.sh} | 15 +++------------ examples/triton-multi-dc/setup-single-dc.sh | 1 + examples/triton/setup.sh | 10 +++++----- 4 files changed, 9 insertions(+), 17 deletions(-) rename examples/{multi-dc => triton-multi-dc}/docker-compose-multi-dc.yml.template (100%) rename examples/{multi-dc/setup-multi-datacenter.sh => triton-multi-dc/setup-multi-dc.sh} (89%) create mode 120000 examples/triton-multi-dc/setup-single-dc.sh diff --git a/examples/multi-dc/docker-compose-multi-dc.yml.template b/examples/triton-multi-dc/docker-compose-multi-dc.yml.template similarity index 100% rename from examples/multi-dc/docker-compose-multi-dc.yml.template rename to examples/triton-multi-dc/docker-compose-multi-dc.yml.template diff --git a/examples/multi-dc/setup-multi-datacenter.sh b/examples/triton-multi-dc/setup-multi-dc.sh similarity index 89% rename from examples/multi-dc/setup-multi-datacenter.sh rename to examples/triton-multi-dc/setup-multi-dc.sh index 2c8aa14..1b98ca8 100755 --- a/examples/multi-dc/setup-multi-datacenter.sh +++ b/examples/triton-multi-dc/setup-multi-dc.sh @@ -57,9 +57,8 @@ INITIAL_PROFILE=$(triton profile get | awk '/name:/{print $2}') for profile in "$@" do echo "Temporarily switching profile: $profile" - triton profile set $profile - eval "$(triton env -d)" - ./setup.sh + eval "TRITON_PROFILE=$profile $(triton env -d)" + ./setup-single-dc.sh unset CONSUL source _env @@ -92,12 +91,4 @@ do written+=("_env-$profile") done -echo "Wrote: $written" - -echo "Clearing triton env" - -eval "$(triton env -u)" -triton profile set "$INITIAL_PROFILE" - -echo "Restored profile: $INITIAL_PROFILE" -echo "You might want to run eval \"\$(triton env) again" \ No newline at end of file +echo "Wrote: ${written[@]}" diff --git a/examples/triton-multi-dc/setup-single-dc.sh b/examples/triton-multi-dc/setup-single-dc.sh new file mode 120000 index 0000000..3c4bb25 --- /dev/null +++ b/examples/triton-multi-dc/setup-single-dc.sh @@ -0,0 +1 @@ +../triton/setup.sh \ No newline at end of file diff --git a/examples/triton/setup.sh b/examples/triton/setup.sh index a196427..895fe92 100755 --- a/examples/triton/setup.sh +++ b/examples/triton/setup.sh @@ -73,12 +73,12 @@ check() { fi # setup environment file - if [ ! -f "examples/triton/_env" ]; then - echo '# Consul bootstrap via Triton CNS' >> examples/triton/_env - echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> examples/triton/_env - echo >> examples/triton/_env + if [ ! -f "_env" ]; then + echo '# Consul bootstrap via Triton CNS' >> _env + echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> _env + echo >> _env else - echo 'Existing _env file found at examples/triton/_env, exiting' + echo 'Existing _env file found at _env, exiting' exit fi } From db96dd747075bf2830027127d9fb087c2c77998b Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 18 Dec 2017 17:15:22 -0800 Subject: [PATCH 22/32] Separate the triton and triton-multi-dc setup scripts --- .gitignore | 2 +- examples/triton-multi-dc/setup-multi-dc.sh | 85 ++++++++++++++++++--- examples/triton-multi-dc/setup-single-dc.sh | 1 - makefile | 4 +- 4 files changed, 79 insertions(+), 13 deletions(-) delete mode 120000 examples/triton-multi-dc/setup-single-dc.sh diff --git a/.gitignore b/.gitignore index ad24b45..b7750de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ _env* -examples/triton/docker-compose-*.yml +examples/triton-multi-dc/docker-compose-*.yml diff --git a/examples/triton-multi-dc/setup-multi-dc.sh b/examples/triton-multi-dc/setup-multi-dc.sh index 1b98ca8..edefdb0 100755 --- a/examples/triton-multi-dc/setup-multi-dc.sh +++ b/examples/triton-multi-dc/setup-multi-dc.sh @@ -18,6 +18,78 @@ if [ "$#" -lt 1 ]; then exit 1 fi +# --------------------------------------------------- +# Top-level commands + +# +# Check for correct configuration and setup _env file named $1 +# +generate_env() { + + command -v docker >/dev/null 2>&1 || { + echo + tput rev # reverse + tput bold # bold + echo 'Docker is required, but does not appear to be installed.' + tput sgr0 # clear + echo 'See https://docs.joyent.com/public-cloud/api-access/docker' + exit 1 + } + command -v triton >/dev/null 2>&1 || { + echo + tput rev # reverse + tput bold # bold + echo 'Error! Joyent Triton CLI is required, but does not appear to be installed.' + tput sgr0 # clear + echo 'See https://www.joyent.com/blog/introducing-the-triton-command-line-tool' + exit 1 + } + + # make sure Docker client is pointed to the same place as the Triton client + local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}') + local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') + + local TRITON_USER=$(triton profile get $TRITON_PROFILE | awk -F": " '/account:/{print $2}') + local TRITON_DC=$(triton profile get $TRITON_PROFILE | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') + local TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}') + + if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then + echo + tput rev # reverse + tput bold # bold + echo 'Error! The Triton CLI configuration does not match the Docker CLI configuration.' + tput sgr0 # clear + echo + echo "Docker user: ${docker_user}" + echo "Triton user: ${TRITON_USER}" + echo "Docker data center: ${docker_dc}" + echo "Triton data center: ${TRITON_DC}" + exit 1 + fi + + local triton_cns_enabled=$(triton account get | awk -F": " '/cns/{print $2}') + if [ ! "true" == "$triton_cns_enabled" ]; then + echo + tput rev # reverse + tput bold # bold + echo 'Error! Triton CNS is required and not enabled.' + tput sgr0 # clear + echo + exit 1 + fi + + # setup environment file + if [ ! -f "$1" ]; then + echo '# Consul bootstrap via Triton CNS' >> $1 + echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> $1 + echo >> $1 + else + echo "Existing _env file found at $1, exiting" + exit + fi +} + + declare -a written declare -a consul_hostnames @@ -51,22 +123,19 @@ if [ ! -f "docker-compose-multi-dc.yml.template" ]; then exit 5 fi -INITIAL_PROFILE=$(triton profile get | awk '/name:/{print $2}') - # invoke ./setup.sh once per profile for profile in "$@" do echo "Temporarily switching profile: $profile" - eval "TRITON_PROFILE=$profile $(triton env -d)" - ./setup-single-dc.sh + eval "$(TRITON_PROFILE=$profile triton env -d)" + + TRITON_PROFILE=$profile generate_env("_env-$profile") unset CONSUL - source _env + source "_env-$profile" consul_hostnames+=("\"${CONSUL//cns.joyent.com/triton.zone}\"") - mv _env "_env-$profile" - cp docker-compose-multi-dc.yml.template \ "docker-compose-$profile.yml" @@ -87,8 +156,6 @@ do "docker-compose-$profile.yml" sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "docker-compose-$profile.yml" - - written+=("_env-$profile") done echo "Wrote: ${written[@]}" diff --git a/examples/triton-multi-dc/setup-single-dc.sh b/examples/triton-multi-dc/setup-single-dc.sh deleted file mode 120000 index 3c4bb25..0000000 --- a/examples/triton-multi-dc/setup-single-dc.sh +++ /dev/null @@ -1 +0,0 @@ -../triton/setup.sh \ No newline at end of file diff --git a/makefile b/makefile index b8b64e8..6260f92 100644 --- a/makefile +++ b/makefile @@ -84,8 +84,8 @@ test/triton/dev: # ------------------------------------------------ # Multi-datacenter usage -clean/multi-dc: - rm -rf examples/multi-dc/_env* examples/multi-dc/docker-compose-*.yml +clean/triton-multi-dc: + rm -rf examples/triton-multi-dc/_env* examples/triton-multi-dc/docker-compose-*.yml ## Print environment for build debugging From eb360fc038a6f39bf1e0b8c0a31b0ad882e913a6 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 18 Dec 2017 17:34:48 -0800 Subject: [PATCH 23/32] make the check function easier to use --- examples/triton-multi-dc/setup-multi-dc.sh | 27 +++++++++++----------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/examples/triton-multi-dc/setup-multi-dc.sh b/examples/triton-multi-dc/setup-multi-dc.sh index edefdb0..2b28528 100755 --- a/examples/triton-multi-dc/setup-multi-dc.sh +++ b/examples/triton-multi-dc/setup-multi-dc.sh @@ -22,9 +22,11 @@ fi # Top-level commands # -# Check for correct configuration and setup _env file named $1 +# Check for triton profile $1 and output _env file named $2 # generate_env() { + local triton_profile=$1 + local output_file=$2 command -v docker >/dev/null 2>&1 || { echo @@ -49,11 +51,11 @@ generate_env() { local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}') local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') - local TRITON_USER=$(triton profile get $TRITON_PROFILE | awk -F": " '/account:/{print $2}') - local TRITON_DC=$(triton profile get $TRITON_PROFILE | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') - local TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}') + local triton_user=$(triton profile get $triton_profile | awk -F": " '/account:/{print $2}') + local triton_dc=$(triton profile get $triton_profile | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') + local triton_account=$(triton account get | awk -F": " '/id:/{print $2}') - if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then + if [ ! "$docker_user" = "$triton_user" ] || [ ! "$docker_dc" = "$triton_dc" ]; then echo tput rev # reverse tput bold # bold @@ -61,9 +63,9 @@ generate_env() { tput sgr0 # clear echo echo "Docker user: ${docker_user}" - echo "Triton user: ${TRITON_USER}" + echo "Triton user: ${triton_user}" echo "Docker data center: ${docker_dc}" - echo "Triton data center: ${TRITON_DC}" + echo "Triton data center: ${triton_dc}" exit 1 fi @@ -79,10 +81,10 @@ generate_env() { fi # setup environment file - if [ ! -f "$1" ]; then - echo '# Consul bootstrap via Triton CNS' >> $1 - echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> $1 - echo >> $1 + if [ ! -f "$output_file" ]; then + echo '# Consul bootstrap via Triton CNS' >> $output_file + echo CONSUL=consul.svc.${triton_account}.${triton_dc}.cns.joyent.com >> $output_file + echo >> $output_file else echo "Existing _env file found at $1, exiting" exit @@ -128,8 +130,7 @@ for profile in "$@" do echo "Temporarily switching profile: $profile" eval "$(TRITON_PROFILE=$profile triton env -d)" - - TRITON_PROFILE=$profile generate_env("_env-$profile") + generate_env $profile "_env-$profile" unset CONSUL source "_env-$profile" From 3c6f336310ccb57d2d558f1b0d7f09f102b2d84a Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Mon, 18 Dec 2017 18:21:57 -0800 Subject: [PATCH 24/32] Fix account id query command --- README.md | 2 +- examples/triton-multi-dc/setup-multi-dc.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 515568d..25aac54 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ When run locally for testing, we don't have access to Triton CNS. The `local-com 1. [Get a Joyent account](https://my.joyent.com/landing/signup/) and [add your SSH key](https://docs.joyent.com/public-cloud/getting-started). 1. Install the [Docker Toolbox](https://docs.docker.com/installation/mac/) (including `docker` and `docker-compose`) on your laptop or other environment, as well as the [Joyent Triton CLI](https://www.joyent.com/blog/introducing-the-triton-command-line-tool) (`triton` replaces our old `sdc-*` CLI tools). -Check that everything is configured correctly by running `./setup.sh`. This will check that your environment is setup correctly and will create an `_env` file that includes injecting an environment variable for a service name for Consul in Triton CNS. We'll use this CNS name to bootstrap the cluster. +Check that everything is configured correctly by changing to the `examples/triton` directory and executing `./setup.sh`. This will check that your environment is setup correctly and will create an `_env` file that includes injecting an environment variable for a service name for Consul in Triton CNS. We'll use this CNS name to bootstrap the cluster. ```bash $ docker-compose up -d diff --git a/examples/triton-multi-dc/setup-multi-dc.sh b/examples/triton-multi-dc/setup-multi-dc.sh index 2b28528..af043a8 100755 --- a/examples/triton-multi-dc/setup-multi-dc.sh +++ b/examples/triton-multi-dc/setup-multi-dc.sh @@ -53,7 +53,7 @@ generate_env() { local triton_user=$(triton profile get $triton_profile | awk -F": " '/account:/{print $2}') local triton_dc=$(triton profile get $triton_profile | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') - local triton_account=$(triton account get | awk -F": " '/id:/{print $2}') + local triton_account=$(TRITON_PROFILE=$triton_profile triton account get | awk -F": " '/id:/{print $2}') if [ ! "$docker_user" = "$triton_user" ] || [ ! "$docker_dc" = "$triton_dc" ]; then echo @@ -125,6 +125,8 @@ if [ ! -f "docker-compose-multi-dc.yml.template" ]; then exit 5 fi +echo "profiles: $@" + # invoke ./setup.sh once per profile for profile in "$@" do From 8b27a61e743b88aba86e499784b94ad3096cbddc Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Tue, 19 Dec 2017 11:03:32 -0800 Subject: [PATCH 25/32] Path and script name change in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 25aac54..867b651 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ $ docker exec -it consul_consul_3 consul info | grep num_peers ### Run it with more than one datacenter! -Follow the same steps as above until you reach `./setup.sh` but execute `./setup-multi-datacenter.sh` instead, providing as arguments Triton profiles which belong to the desired data centers. +Within the `examples/triton-multi-dc` directory, execute `./setup-multi-dc.sh`, providing as arguments Triton profiles which belong to the desired data centers. Since interacting with multiple data centers requires switching between Triton profiles it's easier to perform the following steps in separate terminals. It is possible to perform all the steps for a single data center and then change profiles. Additionally, setting `COMPOSE_PROJECT_NAME` to match the profile or data center will help distinguish nodes in Triton Portal and the `triton instance ls` listing. From 06bc9eaa4fe6d66ba5e0eb3b6ec855e0162b1aa2 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Tue, 19 Dec 2017 11:14:36 -0800 Subject: [PATCH 26/32] Clarify README --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 867b651..fef5b90 100644 --- a/README.md +++ b/README.md @@ -54,18 +54,19 @@ $ docker exec -it consul_consul_3 consul info | grep num_peers ### Run it with more than one datacenter! -Within the `examples/triton-multi-dc` directory, execute `./setup-multi-dc.sh`, providing as arguments Triton profiles which belong to the desired data centers. +Within the `examples/triton-multi-dc` directory, execute `./setup-multi-dc.sh`, providing as arguments Triton profiles which belong to the desired data centers. Since interacting with multiple data centers requires switching between Triton profiles it's easier to perform the following steps in separate terminals. It is possible to perform all the steps for a single data center and then change profiles. Additionally, setting `COMPOSE_PROJECT_NAME` to match the profile or data center will help distinguish nodes in Triton Portal and the `triton instance ls` listing. -Execute the following commands, once per data center, from the project root: +One `_env` and one `docker-compose-.yml` should be generated for each profile. Execute the following commands, once for each profile/datacenter, within `examples/triton-multi-dc`: ``` $ eval "$(TRITON_PROFILE= triton env -d)" -$ export COMPOSE_FILE=examples/triton/docker-compose-.yml +# The following helps when executing docker-compose multiple times. Alternatively, pass the -f flag to each invocation of docker-compose. +$ export COMPOSE_FILE=docker-compose-.yml -# the following is not strictly necessary but helps to discern between clusters +# The following is not strictly necessary but helps to discern between clusters. Alternatively, pass the -p flag to each invocation of docker-compose. $ export COMPOSE_PROJECT_NAME= $ docker-compose up -d From 6759213cd67a187e89808b147d1b30244a29efd0 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Tue, 19 Dec 2017 17:36:33 -0800 Subject: [PATCH 27/32] Fix duplicate env vars from merge conflict resolution --- README.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/README.md b/README.md index fef5b90..0f0b5c7 100644 --- a/README.md +++ b/README.md @@ -136,20 +136,6 @@ services: In our experience, including a Consul cluster within a project's `docker-compose.yml` can help developers understand and test how a service should be discovered and registered within a wider infrastructure context. -#### Environment Variables - -- `CONSUL_DEV`: Enable development mode, allowing a node to self-elect as a cluster leader. Consul flag: [`-dev`](https://www.consul.io/docs/agent/options.html#_dev). - - The following errors will occur if `CONSUL_DEV` is omitted and not enough Consul instances are deployed: - ``` - [ERR] agent: failed to sync remote state: No cluster leader - [ERR] agent: failed to sync changes: No cluster leader - [ERR] agent: Coordinate update error: No cluster leader - ``` -- `CONSUL_DATACENTER_NAME`: Explicitly set the name of the data center in which Consul is running. Consul flag: [`-datacenter`](https://www.consul.io/docs/agent/options.html#datacenter). - - If this variable is specified it will be used as-is. - - If not specified, automatic detection of the datacenter will be attempted. See [issue #23](https://github.com/autopilotpattern/consul/issues/23) for more details. - - Consul's default of "dc1" will be used if none of the above apply. - ### Clients ContainerPilot utilizes Consul's [HTTP Agent API](https://www.consul.io/api/agent.html) for a handful of endpoints, such as `UpdateTTL`, `CheckRegister`, `ServiceRegister` and `ServiceDeregister`. Connecting ContainerPilot to Consul can be achieved by running Consul as a client to a cluster (mentioned above). It's easy to run this Consul client agent from ContainerPilot itself. From 9ea5646ab87f290f66225e0d3e23b39db3e7efd8 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Wed, 20 Dec 2017 14:30:01 -0800 Subject: [PATCH 28/32] Fix usage output description --- examples/triton-multi-dc/setup-multi-dc.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/examples/triton-multi-dc/setup-multi-dc.sh b/examples/triton-multi-dc/setup-multi-dc.sh index af043a8..3d95020 100755 --- a/examples/triton-multi-dc/setup-multi-dc.sh +++ b/examples/triton-multi-dc/setup-multi-dc.sh @@ -5,12 +5,8 @@ help() { echo echo 'Usage ./setup-multi-datacenter.sh [ [...]]' echo - echo 'Invokes ./setup repeatedly to create one _env file per datacenter per triton profile,' - echo 'attempting to preserve the triton profile set before this script was invoked.' - echo - echo 'Warning: The current triton profile will be changed for each invocation of ./setup.sh,' - echo 'this may cause unexpected behavior if other commands that read the current triton profile' - echo 'are executed concurrently!' + echo 'Invokes ./setup repeatedly to create one _env file per triton profile, each of which' + echo 'is presumably associated with a different datacenter.' } if [ "$#" -lt 1 ]; then From ecc75ed9cd40dd879e385bfe4683f7ac7bea9626 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Wed, 20 Dec 2017 16:06:27 -0800 Subject: [PATCH 29/32] Fix description again --- examples/triton-multi-dc/setup-multi-dc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/triton-multi-dc/setup-multi-dc.sh b/examples/triton-multi-dc/setup-multi-dc.sh index 3d95020..97e9919 100755 --- a/examples/triton-multi-dc/setup-multi-dc.sh +++ b/examples/triton-multi-dc/setup-multi-dc.sh @@ -5,7 +5,7 @@ help() { echo echo 'Usage ./setup-multi-datacenter.sh [ [...]]' echo - echo 'Invokes ./setup repeatedly to create one _env file per triton profile, each of which' + echo 'Generates one _env file and docker-compose.yml file per triton profile, each of which' echo 'is presumably associated with a different datacenter.' } From 658d4e7a048a167a48b7a3894175f7fd780f4378 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Wed, 20 Dec 2017 16:44:49 -0800 Subject: [PATCH 30/32] Remove the build key from examples/compose/docker-compose.yml --- examples/compose/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/compose/docker-compose.yml b/examples/compose/docker-compose.yml index b90dab9..60133f0 100644 --- a/examples/compose/docker-compose.yml +++ b/examples/compose/docker-compose.yml @@ -9,7 +9,6 @@ services: # env var) to find each other and bootstrap the cluster. # Note: Unless CONSUL_DEV is set, at least three instances are required for quorum. consul: - build: ../../ image: autopilotpattern/consul:${TAG:-latest} restart: always mem_limit: 128m From 4cf52d9bee57430275a0d364ba984e83d67525f2 Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Thu, 21 Dec 2017 12:54:22 -0800 Subject: [PATCH 31/32] Revert ./setup.sh change --- examples/triton/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/triton/setup.sh b/examples/triton/setup.sh index 895fe92..da80a57 100755 --- a/examples/triton/setup.sh +++ b/examples/triton/setup.sh @@ -43,8 +43,8 @@ check() { local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}') local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') - TRITON_USER=$(triton profile get $TRITON_PROFILE | awk -F": " '/account:/{print $2}') - TRITON_DC=$(triton profile get $TRITON_PROFILE | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') + TRITON_USER=$(triton profile get | awk -F": " '/account:/{print $2}') + TRITON_DC=$(triton profile get | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}') if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then From dc5018fac8b1fa2c30c5f546ee702a4728145c2f Mon Sep 17 00:00:00 2001 From: Tomas Celaya Date: Thu, 21 Dec 2017 12:54:52 -0800 Subject: [PATCH 32/32] Typo --- examples/triton/setup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/triton/setup.sh b/examples/triton/setup.sh index da80a57..b7e7687 100755 --- a/examples/triton/setup.sh +++ b/examples/triton/setup.sh @@ -46,7 +46,6 @@ check() { TRITON_USER=$(triton profile get | awk -F": " '/account:/{print $2}') TRITON_DC=$(triton profile get | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}') - if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then echo tput rev # reverse @@ -78,7 +77,7 @@ check() { echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> _env echo >> _env else - echo 'Existing _env file found at _env, exiting' + echo 'Existing _env file found, exiting' exit fi }