Add Metricbeat extension (#567)

Co-authored-by: Antoine Cotten <hello@acotten.com>

.github/workflows/ci.yml

@@ -53,6 +53,7 @@ jobs:
           sed -i -e 's/\(elasticsearch.username:\) elastic/\1 kibana_system/g' -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml
           sed -i -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' -e 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml
           sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/apm-server/config/apm-server.yml
+          sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/metricbeat/config/metricbeat.yml
 
           # Run Elasticsearch and wait for its availability
 
@@ -174,6 +175,23 @@ jobs:
           # next steps don't need APM Server
           docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml stop apm-server
 
+      #
+      # Metricbeat
+      #
+
+      - name: Execute Metricbeat test suite
+        run: |
+          docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up -d metricbeat
+          .github/workflows/scripts/run-tests-metricbeat.sh
+
+      - name: 'debug: Display state and logs (Metricbeat)'
+        if: always()
+        run: |
+          docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml ps
+          docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml logs metricbeat
+          # next steps don't need Metricbeat
+          docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml stop metricbeat
+
         ##############
         #            #
         # Tear down. #
@@ -188,6 +206,7 @@ jobs:
           -f extensions/logspout/logspout-compose.yml
           -f extensions/enterprise-search/enterprise-search-compose.yml
           -f extensions/apm-server/apm-server-compose.yml
+          -f extensions/metricbeat/metricbeat-compose.yml
           down -v
 
   test-swarm:

.github/workflows/scripts/run-tests-metricbeat.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+
+source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+
+
+cid_es="$(container_id elasticsearch)"
+cid_mb="$(container_id metricbeat)"
+
+ip_es="$(service_ip elasticsearch)"
+ip_mb="$(service_ip metricbeat)"
+
+log 'Waiting for readiness of Elasticsearch'
+poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+
+log 'Waiting for readiness of Metricbeat'
+poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty"
+
+# We expect to find one monitoring entry for the 'elasticsearch' Compose
+# service using the following query:
+#
+#   agent.type:"metricbeat"
+#   AND event.module:"docker"
+#   AND event.dataset:"docker.container"
+#   AND container.name:"docker-elk_elasticsearch_1"
+#
+log 'Searching a document generated by Metricbeat'
+
+declare response
+declare -i count
+
+# retry for max 60s (30*2s)
+for _ in $(seq 1 30); do
+	response="$(curl "http://${ip_es}:9200/metricbeat-*/_search?q=agent.type:%22metricbeat%22%20AND%20event.module:%22docker%22%20AND%20event.dataset:%22docker.container%22%20AND%20container.name:%22docker-elk_elasticsearch_1%22&pretty" -s -u elastic:testpasswd)"
+	count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
+	if (( count > 0 )); then
+		break
+	fi
+
+	echo -n 'x' >&2
+	sleep 2
+done
+echo -e '\n' >&2
+
+echo "$response"
+if (( count != 1 )); then
+	echo "Expected 1 document, got ${count}"
+	exit 1
+fi