Update to v8.0.0 (#544)

List of changes impacting docker-elk:

- [logstash]: The output to Elasticsearch is handled as a data stream.

  Starting with v8.0.0, the `elasticsearch` output for Logstash sends
  log data to a data stream instead of `logstash-*` indices by default.
  The name of the default data stream is `logs-generic-default`.
  docker-elk remains unopinionated and simply uses Elastic's defaults
  like it always has, so users who prefer to retain the old behaviour
  need to explicitly opt-out of data streams in their Logstash
  pipelines.

  Refs:
  - https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams.html
  - https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams

- [logstash]: The (legacy) monitoring data collection is now disabled.

  This feature was deprecated since v7.9.0, and removed in v8.0.0.

  Ref: https://www.elastic.co/guide/en/logstash/current/monitoring-internal-collection-legacy.html

- [kibana]: An index pattern for `logs-*` indices is automatically
  created.

  It used to be required to manually create an index pattern for indices
  managed by Logstash, even when using the default Logstash indices.
  This is no longer the case since the output data is now being handled
  as a data stream, and Kibana automatically creates index patterns for
  these.

- [elasticsearch]: The command line tool `elasticsearch-setup-passwords`
  was deprecated in favour of a new `elasticsearch-reset-password` tool.

  Passwords for built-in users must now be generated one by one.

  Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-passwords.html

- [enterprise-search]: Kibana is now the new management interface, and
  the only one available moving forward.

  The old standalone Enterprise Search interface was removed in v8.0.0.

  Ref: https://www.elastic.co/guide/en/enterprise-search/current/user-interfaces.html

.github/workflows/scripts/elasticsearch-setup-passwords.exp

@@ -1,7 +1,7 @@
 #!/usr/bin/expect -f
 
 # List of expected users with dummy password
-set user "(elastic|apm_system|kibana_system|logstash_system|beats_system|remote_monitoring_user)"
+set users {"elastic" "kibana_system" "logstash_system" "beats_system" "apm_system" "remote_monitoring_user"}
 set password "testpasswd"
 
 # Find elasticsearch container id
@@ -12,17 +12,27 @@ if { [string match "swarm" $MODE] } {
 	set cid [exec docker ps -q -f label=com.docker.compose.service=elasticsearch]
 }
 
-set cmd "docker exec -it $cid bin/elasticsearch-setup-passwords interactive -s -b -u http://localhost:9200"
+foreach user $users {
+	set cmd "docker exec -it $cid bin/elasticsearch-reset-password --batch --user $user -i"
 
-spawn {*}$cmd
+	spawn {*}$cmd
 
-expect {
-	-re "(E|Ree)nter password for \\\[$user\\\]: " {
-		send "$password\r"
-		exp_continue
+	expect {
+		-re "(E|Re-e)nter password for \\\[$user\\\]: " {
+			send "$password\r"
+			exp_continue
+		}
+		timeout {
+			puts "\ntimed out waiting for input"
+			exit 4
+		}
+		eof
 	}
-	eof
-}
 
-lassign [wait] pid spawnid os_error_flag value
-exit $value
+	lassign [wait] pid spawnid os_error_flag value
+
+	if {$value != 0} {
+		if {$os_error_flag == 0} { puts "exit status: $value" } else { puts "errno: $value" }
+		exit $value
+	}
+}

.github/workflows/scripts/run-tests-core.sh

@@ -24,25 +24,6 @@ poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty"
 log 'Waiting for readiness of Kibana'
 poll_ready "$cid_kb" "http://${ip_kb}:5601/api/status" -u 'kibana_system:testpasswd'
 
-log 'Creating Logstash index pattern in Kibana'
-source .env
-curl -X POST -D- "http://${ip_kb}:5601/api/saved_objects/index-pattern" \
-	-s -w '\n' \
-	-H 'Content-Type: application/json' \
-	-H "kbn-version: ${ELK_VERSION}" \
-	-u elastic:testpasswd \
-	-d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}'
-
-log 'Searching index pattern via Kibana API'
-response="$(curl "http://${ip_kb}:5601/api/saved_objects/_find?type=index-pattern" -s -u elastic:testpasswd)"
-echo "$response"
-declare -i count
-count="$(jq -rn --argjson data "${response}" '$data.total')"
-if (( count != 1 )); then
-	echo "Expected 1 index pattern, got ${count}"
-	exit 1
-fi
-
 log 'Sending message to Logstash TCP input'
 
 declare -i was_retried=0
@@ -62,13 +43,14 @@ if ((was_retried)); then
 	echo >&2
 fi
 
-sleep 3
-curl -X POST "http://${ip_es}:9200/_refresh" -u elastic:testpasswd \
+sleep 5
+curl -X POST "http://${ip_es}:9200/logs-generic-default/_refresh" -u elastic:testpasswd \
 	-s -w '\n'
 
 log 'Searching message in Elasticsearch'
-response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=message:dockerelk&pretty" -s -u elastic:testpasswd)"
+response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=message:dockerelk&pretty" -s -u elastic:testpasswd)"
 echo "$response"
+declare -i count
 count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
 if (( count != 1 )); then
 	echo "Expected 1 document, got ${count}"

.github/workflows/scripts/run-tests-logspout.sh

@@ -39,7 +39,7 @@ declare -i was_retried=0
 
 # retry for max 60s (30*2s)
 for _ in $(seq 1 30); do
-	response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)"
+	response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)"
 
 	set +u  # prevent "unbound variable" if assigned value is not an integer
 	count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"