diff --git a/.env b/.env index 68eb612..1ea22ef 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -ELASTIC_VERSION=8.17.4 +ELASTIC_VERSION=9.0.0 ## Passwords for stack users # diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6eb04f3..b56f3e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,6 @@ jobs: -f extensions/metricbeat/metricbeat-compose.yml -f extensions/filebeat/filebeat-compose.yml -f extensions/heartbeat/heartbeat-compose.yml - -f extensions/enterprise-search/enterprise-search-compose.yml build ######################################################## @@ -58,9 +57,7 @@ jobs: .env - name: Generate Kibana encryption keys - run: | - docker container run --rm docker-elk-kibana bin/kibana-encryption-keys generate -q >>kibana/config/kibana.yml - sed -i '/Kibana is currently running with legacy OpenSSL providers enabled/d' kibana/config/kibana.yml + run: docker container run --rm docker-elk-kibana bin/kibana-encryption-keys generate -q >>kibana/config/kibana.yml - name: Set up users and roles run: docker compose up setup @@ -123,33 +120,6 @@ jobs: docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml up --remove-orphans -d heartbeat .github/workflows/scripts/run-tests-heartbeat.sh - # - # Enterprise Search - # - - - name: Execute Enterprise Search test suite - run: | - - # Set mandatory Elasticsearch settings - - sed -i '$ a xpack.security.authc.api_key.enabled: true' elasticsearch/config/elasticsearch.yml - - # Restart Elasticsearch for changes to take effect - - docker compose restart elasticsearch - - # Run Enterprise Search and execute tests - - sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml - - docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up --remove-orphans -d enterprise-search - .github/workflows/scripts/run-tests-enterprise-search.sh - - # Revert changes to Elasticsearch configuration - - sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml - docker compose restart elasticsearch - - name: Collect troubleshooting data id: debug-data if: failure() @@ -163,7 +133,6 @@ jobs: -f extensions/metricbeat/metricbeat-compose.yml \ -f extensions/filebeat/filebeat-compose.yml \ -f extensions/heartbeat/heartbeat-compose.yml \ - -f extensions/enterprise-search/enterprise-search-compose.yml \ ps >"$debug_data_dir"/docker_ps.log docker compose \ @@ -173,7 +142,6 @@ jobs: -f extensions/metricbeat/metricbeat-compose.yml \ -f extensions/filebeat/filebeat-compose.yml \ -f extensions/heartbeat/heartbeat-compose.yml \ - -f extensions/enterprise-search/enterprise-search-compose.yml \ logs >"$debug_data_dir"/docker_logs.log echo "path=${debug_data_dir}" >>"$GITHUB_OUTPUT" @@ -201,5 +169,4 @@ jobs: -f extensions/metricbeat/metricbeat-compose.yml -f extensions/filebeat/filebeat-compose.yml -f extensions/heartbeat/heartbeat-compose.yml - -f extensions/enterprise-search/enterprise-search-compose.yml down -v diff --git a/.github/workflows/scripts/run-tests-enterprise-search.sh b/.github/workflows/scripts/run-tests-enterprise-search.sh deleted file mode 100755 index 21aa8ee..0000000 --- a/.github/workflows/scripts/run-tests-enterprise-search.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -set -eu -set -o pipefail - - -source "${BASH_SOURCE[0]%/*}"/lib/testing.sh - - -cid_es="$(container_id elasticsearch)" -cid_en="$(container_id enterprise-search)" - -ip_es="$(service_ip elasticsearch)" -ip_en="$(service_ip enterprise-search)" - -grouplog 'Wait for readiness of Elasticsearch' -poll_ready "$cid_es" 'http://elasticsearch:9200/' --resolve "elasticsearch:9200:${ip_es}" -u 'elastic:testpasswd' -endgroup - -grouplog 'Wait for readiness of Enterprise Search' -poll_ready "$cid_en" 'http://enterprise-search:3002/api/ent/v1/internal/health' --resolve "enterprise-search:3002:${ip_en}" -u 'elastic:testpasswd' -endgroup - -log 'Ensuring that App Search API keys were created in Elasticsearch' - -query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) < [!NOTE] -> To generate a strong random encryption key, you can use the OpenSSL utility or any other online/offline tool of your -> choice: -> -> ```console -> $ openssl rand -hex 32 -> 680f94e568c90364bedf927b2f0f49609702d3eab9098688585a375b14274546 -> ``` - -### Enable Elasticsearch's API key service - -Enterprise Search requires Elasticsearch's built-in [API key service][es-security] to be enabled in order to start. -Unless Elasticsearch is configured to enable TLS on the HTTP interface (disabled by default), this service is disabled -by default. - -To enable it, modify the Elasticsearch configuration file in [`elasticsearch/config/elasticsearch.yml`][config-es] and -add the following setting: - -```yaml -xpack.security.authc.api_key.enabled: true -``` - -### Configure the Enterprise Search host in Kibana - -Kibana acts as the [management interface][enterprisesearch-kb] to Enterprise Search. - -To enable the management experience for Enterprise Search, modify the Kibana configuration file in -[`kibana/config/kibana.yml`][config-kbn] and add the following setting: - -```yaml -enterpriseSearch.host: http://enterprise-search:3002 -``` - -### Start the server - -To include Enterprise Search in the stack, run Docker Compose from the root of the repository with an additional command -line argument referencing the `enterprise-search-compose.yml` file: - -```console -$ docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up -``` - -Allow a few minutes for the stack to start, then open your web browser at the address to see the -Enterprise Search home page. - -Enterprise Search is configured on first boot with the following default credentials: - -* user: *enterprise_search* -* password: *changeme* - -## Security - -The Enterprise Search password is defined inside the Compose file via the `ENT_SEARCH_DEFAULT_PASSWORD` environment -variable. We highly recommend choosing a more secure password than the default one for security reasons. - -To do so, change the value `ENT_SEARCH_DEFAULT_PASSWORD` environment variable inside the Compose file **before the first -boot**: - -```yaml -enterprise-search: - - environment: - ENT_SEARCH_DEFAULT_PASSWORD: {{some strong password}} -``` - -> [!WARNING] -> The default Enterprise Search password can only be set during the initial boot. Once the password is persisted in -> Elasticsearch, it can only be changed via the Elasticsearch API. - -For more information, please refer to [User Management and Security][enterprisesearch-security]. - -## Configuring Enterprise Search - -The Enterprise Search configuration is stored in [`config/enterprise-search.yml`][config-enterprisesearch]. You can -modify this file using the [Default Enterprise Search configuration][enterprisesearch-config] as a reference. - -You can also specify the options you want to override by setting environment variables inside the Compose file: - -```yaml -enterprise-search: - - environment: - ent_search.auth.source: standard - worker.threads: '6' -``` - -Any change to the Enterprise Search configuration requires a restart of the Enterprise Search container: - -```console -$ docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml restart enterprise-search -``` - -Please refer to the following documentation page for more details about how to configure Enterprise Search inside a -Docker container: [Running Enterprise Search Using Docker][enterprisesearch-docker]. - -## See also - -[Enterprise Search documentation][enterprisesearch-docs] - -[config-enterprisesearch]: ./config/enterprise-search.yml - -[enterprisesearch-encryption]: https://www.elastic.co/guide/en/enterprise-search/current/encryption-keys.html -[enterprisesearch-security]: https://www.elastic.co/guide/en/workplace-search/current/workplace-search-security.html -[enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html -[enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html -[enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html -[enterprisesearch-kb]: https://www.elastic.co/guide/en/kibana/current/enterprise-search-settings-kb.html - -[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings -[config-es]: ../../elasticsearch/config/elasticsearch.yml -[config-kbn]: ../../kibana/config/kibana.yml diff --git a/extensions/enterprise-search/config/enterprise-search.yml b/extensions/enterprise-search/config/enterprise-search.yml deleted file mode 100644 index a1f098d..0000000 --- a/extensions/enterprise-search/config/enterprise-search.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -## Enterprise Search core configuration -## https://www.elastic.co/guide/en/enterprise-search/current/configuration.html -# - -## --------------------- REQUIRED --------------------- - -# Encryption keys to protect application secrets. -secret_management.encryption_keys: - # example: - #- 680f94e568c90364bedf927b2f0f49609702d3eab9098688585a375b14274546 - -## ---------------------------------------------------- - -# IP address Enterprise Search listens on -ent_search.listen_host: 0.0.0.0 - -# URL at which users reach Enterprise Search / Kibana -ent_search.external_url: http://localhost:3002 -kibana.host: http://localhost:5601 - -# Elasticsearch URL and credentials -elasticsearch.host: http://elasticsearch:9200 -elasticsearch.username: elastic -elasticsearch.password: ${ELASTIC_PASSWORD} - -# Allow Enterprise Search to modify Elasticsearch settings. Used to enable auto-creation of Elasticsearch indexes. -allow_es_settings_modification: true diff --git a/extensions/enterprise-search/enterprise-search-compose.yml b/extensions/enterprise-search/enterprise-search-compose.yml deleted file mode 100644 index 84a0011..0000000 --- a/extensions/enterprise-search/enterprise-search-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -services: - enterprise-search: - build: - context: extensions/enterprise-search/ - args: - ELASTIC_VERSION: ${ELASTIC_VERSION} - volumes: - - ./extensions/enterprise-search/config/enterprise-search.yml:/usr/share/enterprise-search/config/enterprise-search.yml:ro,Z - environment: - JAVA_OPTS: -Xms2g -Xmx2g - ENT_SEARCH_DEFAULT_PASSWORD: 'changeme' - ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} - ports: - - 3002:3002 - networks: - - elk - depends_on: - - elasticsearch diff --git a/extensions/filebeat/Dockerfile b/extensions/filebeat/Dockerfile index 792d9e1..6a64970 100644 --- a/extensions/filebeat/Dockerfile +++ b/extensions/filebeat/Dockerfile @@ -1,3 +1,3 @@ ARG ELASTIC_VERSION -FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION:-8.17.4} +FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION:-9.0.0} diff --git a/extensions/fleet/Dockerfile b/extensions/fleet/Dockerfile index bc98916..93f614f 100644 --- a/extensions/fleet/Dockerfile +++ b/extensions/fleet/Dockerfile @@ -1,6 +1,6 @@ ARG ELASTIC_VERSION -FROM docker.elastic.co/elastic-agent/elastic-agent:${ELASTIC_VERSION:-8.17.4} +FROM docker.elastic.co/elastic-agent/elastic-agent:${ELASTIC_VERSION:-9.0.0} # Ensure the 'state' directory exists and is owned by the 'elastic-agent' user, # otherwise mounting a named volume in that location creates a directory owned diff --git a/extensions/heartbeat/Dockerfile b/extensions/heartbeat/Dockerfile index 32bfc83..0500ee3 100644 --- a/extensions/heartbeat/Dockerfile +++ b/extensions/heartbeat/Dockerfile @@ -1,3 +1,3 @@ ARG ELASTIC_VERSION -FROM docker.elastic.co/beats/heartbeat:${ELASTIC_VERSION:-8.17.4} +FROM docker.elastic.co/beats/heartbeat:${ELASTIC_VERSION:-9.0.0} diff --git a/extensions/metricbeat/Dockerfile b/extensions/metricbeat/Dockerfile index 1986606..81ba61b 100644 --- a/extensions/metricbeat/Dockerfile +++ b/extensions/metricbeat/Dockerfile @@ -1,3 +1,3 @@ ARG ELASTIC_VERSION -FROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION:-8.17.4} +FROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION:-9.0.0} diff --git a/kibana/Dockerfile b/kibana/Dockerfile index d581ce4..c193058 100644 --- a/kibana/Dockerfile +++ b/kibana/Dockerfile @@ -1,7 +1,7 @@ ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION:-8.17.4} +FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION:-9.0.0} # Add your kibana plugins setup here # Example: RUN kibana-plugin install diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index ef3f024..e295087 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -17,7 +17,7 @@ elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD} ## Encryption keys (optional but highly recommended) ## ## Generate with either -## $ docker container run --rm docker.elastic.co/kibana/kibana:8.6.2 bin/kibana-encryption-keys generate +## $ docker container run --rm docker.elastic.co/kibana/kibana:9.0.0 bin/kibana-encryption-keys generate ## $ openssl rand -hex 32 ## ## https://www.elastic.co/guide/en/kibana/current/using-kibana-with-security.html diff --git a/logstash/Dockerfile b/logstash/Dockerfile index 6ee0300..5425610 100644 --- a/logstash/Dockerfile +++ b/logstash/Dockerfile @@ -1,7 +1,7 @@ ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-8.17.4} +FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-9.0.0} # Add your logstash plugins setup here # Example: RUN logstash-plugin install logstash-filter-json diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml index a81b89b..06b05d7 100644 --- a/logstash/config/logstash.yml +++ b/logstash/config/logstash.yml @@ -2,6 +2,6 @@ ## Default Logstash configuration from Logstash base image. ## https://github.com/elastic/logstash/blob/main/docker/data/logstash/config/logstash-full.yml # -http.host: 0.0.0.0 +api.http.host: 0.0.0.0 node.name: logstash diff --git a/setup/Dockerfile b/setup/Dockerfile index 20b32f1..7071424 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -1,6 +1,6 @@ ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-8.17.4} +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-9.0.0} ENTRYPOINT ["/entrypoint.sh"]