Move CI flow from Travis CI to GitHub Actions

.github/workflows/scripts/elasticsearch-setup-passwords.exp

@@ -0,0 +1,28 @@
+#!/usr/bin/expect -f
+
+# List of expected users with dummy password
+set user "(elastic|apm_system|kibana_system|logstash_system|beats_system|remote_monitoring_user)"
+set password "testpasswd"
+
+# Find elasticsearch container id
+set MODE [lindex $argv 0]
+if { [string match "swarm" $MODE] } {
+	set cid [exec docker ps -q -f label=com.docker.swarm.service.name=elk_elasticsearch]
+} else {
+	set cid [exec docker ps -q -f label=com.docker.compose.service=elasticsearch]
+}
+
+set cmd "docker exec -it $cid bin/elasticsearch-setup-passwords interactive -s -b"
+
+spawn {*}$cmd
+
+expect {
+	-re "(E|Ree)nter password for \\\[$user\\\]: " {
+		send "$password\r"
+		exp_continue
+	}
+	eof
+}
+
+lassign [wait] pid spawnid os_error_flag value
+exit $value

.github/workflows/scripts/lib/testing.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+# Log a message.
+function log {
+	echo -e "\n[+] $1\n"
+}
+
+# Log an error.
+function err {
+	echo -e "\n[x] $1\n"
+}
+
+# Return the ID of the container running the given service.
+function container_id {
+	local svc=$1
+
+	local label
+	if [[ "$MODE" == "swarm" ]]; then
+		label="com.docker.swarm.service.name=elk_${svc}"
+	else
+		label="com.docker.compose.service=${svc}"
+	fi
+
+	local cid
+	# retry for max 60s (30*2s)
+	for _ in $(seq 1 30); do
+		cid="$(docker container ls -aq -f label="$label")"
+		if [ -n "$cid" ]; then
+			break
+		fi
+
+		echo -n '.' >&2
+		sleep 2
+	done
+	echo -e '\n' >&2
+
+	if [ -z "${cid:-}" ]; then
+		err "Timed out waiting for creation of container with label ${label}"
+		return 1
+	fi
+
+	echo "$cid"
+}
+
+# Return the IP address at which a service can be reached.
+# In Compose mode, returns the container's IP.
+# In Swarm mode, returns the IP of the node to ensure traffic enters the routing mesh (ingress).
+function service_ip {
+	local svc=$1
+
+	local ip
+
+	if [[ "$MODE" == "swarm" ]]; then
+		#ingress_net="$(docker network inspect ingress --format '{{ .Id }}')"
+		#ip="$(docker service inspect elk_"$svc" --format "{{ range .Endpoint.VirtualIPs }}{{ if eq .NetworkID \"${ingress_net}\" }}{{ .Addr }}{{ end }}{{ end }}" | cut -d/ -f1)"
+		node="$(docker node ls --format '{{ .ID }}')"
+		ip="$(docker node inspect "$node" --format '{{ .Status.Addr }}')"
+		if [ -z "${ip:-}" ]; then
+			err "Node ${node} has no IP address"
+			return 1
+		fi
+
+		echo "$ip"
+		return
+	fi
+
+	local cid
+	cid="$(container_id "$svc")"
+
+	ip="$(docker container inspect "$cid" --format '{{ (index .NetworkSettings.Networks "docker-elk_elk").IPAddress }}')"
+	if [ -z "${ip:-}" ]; then
+		err "Container ${cid} has no IP address"
+		return 1
+	fi
+
+	echo "$ip"
+}
+
+# Poll the given service at the given port:/path until it responds with HTTP code 200.
+function poll_ready {
+	local cid=$1
+	local url=$2
+
+	local -a args=( '-s' '-D-' '-m3' '-w' '%{http_code}' "$url" )
+	if [ "$#" -ge 3 ]; then
+		args+=( '-u' "$3" )
+	fi
+
+	echo "curl arguments: ${args[*]}"
+
+	local -i result=1
+	local output
+
+	# retry for max 180s (36*5s)
+	for _ in $(seq 1 36); do
+		if [[ $(docker container inspect "$cid" --format '{{ .State.Status}}') == 'exited' ]]; then
+			err "Container exited ($(docker container inspect "$cid" --format '{{ .Name }}'))"
+			return 1
+		fi
+
+		output="$(curl "${args[@]}" || true)"
+		if [ "${output: -3}" -eq 200 ]; then
+			result=0
+			break
+		fi
+
+		echo -n 'x' >&2
+		sleep 5
+	done
+	echo -e '\n' >&2
+
+	echo -e "\n${output::-3}"
+
+	return $result
+}

.github/workflows/scripts/run-tests-apm-server.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+
+source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+
+
+declare MODE=""
+
+
+cid="$(container_id apm-server)"
+ip="$(service_ip apm-server)"
+
+log 'Waiting for readiness of APM Server'
+poll_ready "$cid" "http://${ip}:8200/"

.github/workflows/scripts/run-tests-core.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+
+source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+
+
+declare MODE=""
+if [ "$#" -ge 1 ]; then
+	MODE=$1
+fi
+
+cid_es="$(container_id elasticsearch)"
+cid_ls="$(container_id logstash)"
+cid_kb="$(container_id kibana)"
+
+ip_es="$(service_ip elasticsearch)"
+ip_ls="$(service_ip logstash)"
+ip_kb="$(service_ip kibana)"
+
+log 'Waiting for readiness of Elasticsearch'
+poll_ready "$cid_es" "http://${ip_es}:9200/" 'elastic:testpasswd'
+
+log 'Waiting for readiness of Logstash'
+poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty"
+
+log 'Waiting for readiness of Kibana'
+poll_ready "$cid_kb" "http://${ip_kb}:5601/api/status" 'kibana_system:testpasswd'
+
+log 'Creating Logstash index pattern in Kibana'
+source .env
+curl -X POST -D- "http://${ip_kb}:5601/api/saved_objects/index-pattern" \
+	-s -w '\n' \
+	-H 'Content-Type: application/json' \
+	-H "kbn-version: ${ELK_VERSION}" \
+	-u elastic:testpasswd \
+	-d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}'
+
+log 'Searching index pattern via Kibana API'
+response="$(curl "http://${ip_kb}:5601/api/saved_objects/_find?type=index-pattern" -s -u elastic:testpasswd)"
+echo "$response"
+count="$(jq -rn --argjson data "${response}" '$data.total')"
+if [[ $count -ne 1 ]]; then
+	echo "Expected 1 index pattern, got ${count}"
+	exit 1
+fi
+
+log 'Sending message to Logstash TCP input'
+echo 'dockerelk' | nc -q0 "$ip_ls" 5000
+
+sleep 1
+curl -X POST "http://${ip_es}:9200/_refresh" -u elastic:testpasswd \
+	-s -w '\n'
+
+log 'Searching message in Elasticsearch'
+response="$(curl "http://${ip_es}:9200/_count?q=message:dockerelk&pretty" -s -u elastic:testpasswd)"
+echo "$response"
+count="$(jq -rn --argjson data "${response}" '$data.count')"
+if [[ $count -ne 1 ]]; then
+	echo "Expected 1 document, got ${count}"
+	exit 1
+fi

.github/workflows/scripts/run-tests-enterprise-search.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+
+source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+
+
+declare MODE=""
+
+
+cid_es="$(container_id elasticsearch)"
+cid_en="$(container_id enterprise-search)"
+
+ip_es="$(service_ip elasticsearch)"
+ip_en="$(service_ip enterprise-search)"
+
+log 'Waiting for readiness of Elasticsearch'
+poll_ready "$cid_es" "http://${ip_es}:9200/" 'elastic:testpasswd'
+
+log 'Waiting for readiness of Enterprise Search'
+poll_ready "$cid_en" "http://${ip_en}:3002/api/ent/v1/internal/health" 'elastic:testpasswd'
+
+log 'Retrieving private key from Elasticsearch'
+response="$(curl "http://${ip_es}:9200/.ent-search-actastic-app_search_api_tokens_v2/_search?q=name:private-key" -s -u elastic:testpasswd)"
+hits="$(jq -rn --argjson data "${response}" '$data.hits.hits')"
+echo "$hits"
+count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
+if [[ $count -ne 1 ]]; then
+	echo "Private key not found. Expected 1 result, got ${count}"
+	exit 1
+fi
+key="$(jq -rn --argjson data "${hits}" '$data[0]._source.authentication_token')"
+
+log 'Creating App Search engine'
+response="$(curl "http://${ip_en}:3002/api/as/v1/engines" -s -d '{"name": "dockerelk"}' -H "Authorization: Bearer ${key}")"
+echo "$response"
+name="$(jq -rn --argjson data "${response}" '$data.name')"
+if [[ $name != 'dockerelk' ]]; then
+	echo 'Failed to create engine'
+	exit 1
+fi