diff --git a/.env b/.env index 7f5df1d..40c80c1 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -ELASTIC_VERSION=8.15.2 +ELASTIC_VERSION=7.17.24 ## Passwords for stack users # diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 97bdac6..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Dependabot configuration -# -# For more information, please refer to: -# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates - -version: 2 - -updates: - -# Maintain dependencies for GitHub Actions -- package-ecosystem: github-actions - directory: / - schedule: - interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 594eb3a..f51c6f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [ main ] + branches: [ release-7.x ] pull_request: - branches: [ main ] + branches: [ release-7.x ] jobs: @@ -124,27 +124,11 @@ jobs: - name: Execute Enterprise Search test suite run: | - - # Set mandatory Elasticsearch settings - - sed -i '$ a xpack.security.authc.api_key.enabled: true' elasticsearch/config/elasticsearch.yml - - # Restart Elasticsearch for changes to take effect - - docker compose restart elasticsearch - - # Run Enterprise Search and execute tests - sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up --remove-orphans -d enterprise-search .github/workflows/scripts/run-tests-enterprise-search.sh - # Revert changes to Elasticsearch configuration - - sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml - docker compose restart elasticsearch - - name: Collect troubleshooting data id: debug-data if: failure() diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 27687d6..77f1020 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,12 +1,10 @@ name: Documentation on: - schedule: - - cron: '0 0 * * 0' # At 00:00 every Sunday push: - branches: [ main ] + branches: [ release-7.x ] pull_request: - branches: [ main ] + branches: [ release-7.x ] jobs: diff --git a/.github/workflows/scripts/elasticsearch-setup-passwords.exp b/.github/workflows/scripts/elasticsearch-setup-passwords.exp new file mode 100755 index 0000000..b0cc497 --- /dev/null +++ b/.github/workflows/scripts/elasticsearch-setup-passwords.exp @@ -0,0 +1,23 @@ +#!/usr/bin/expect -f + +# List of expected users with dummy password +set user "(elastic|apm_system|kibana_system|logstash_system|beats_system|remote_monitoring_user)" +set password "testpasswd" + +# Find elasticsearch container id +set cid [exec docker ps -q -f label=com.docker.compose.service=elasticsearch] + +set cmd "docker exec -it $cid bin/elasticsearch-setup-passwords interactive -s -b -u http://localhost:9200" + +spawn {*}$cmd + +expect { + -re "(E|Ree)nter password for \\\[$user\\\]: " { + send "$password\r" + exp_continue + } + eof +} + +lassign [wait] pid spawnid os_error_flag value +exit $value diff --git a/.github/workflows/scripts/run-tests-core.sh b/.github/workflows/scripts/run-tests-core.sh index ab79df1..faf02aa 100755 --- a/.github/workflows/scripts/run-tests-core.sh +++ b/.github/workflows/scripts/run-tests-core.sh @@ -27,6 +27,25 @@ grouplog 'Wait for readiness of Kibana' poll_ready "$cid_kb" 'http://kibana:5601/api/status' --resolve "kibana:5601:${ip_kb}" -u 'kibana_system:testpasswd' endgroup +log 'Creating Logstash index pattern in Kibana' +source .env +curl -X POST -D- "http://${ip_kb}:5601/api/saved_objects/index-pattern" \ + -s -w '\n' \ + -H 'Content-Type: application/json' \ + -H "kbn-version: ${ELASTIC_VERSION}" \ + -u elastic:testpasswd \ + -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' + +log 'Searching index pattern via Kibana API' +response="$(curl "http://${ip_kb}:5601/api/saved_objects/_find?type=index-pattern" -s -u elastic:testpasswd)" +echo "$response" +declare -i count +count="$(jq -rn --argjson data "${response}" '$data.total')" +if (( count != 1 )); then + echo "Expected 1 index pattern, got ${count}" + exit 1 +fi + log 'Sending message to Logstash TCP input' declare -i was_retried=0 @@ -47,7 +66,7 @@ if ((was_retried)); then fi declare -a refresh_args=( '-X' 'POST' '-s' '-w' '%{http_code}' '-u' 'elastic:testpasswd' - 'http://elasticsearch:9200/logs-generic-default/_refresh' + 'http://elasticsearch:9200/logstash-*/_refresh' '--resolve' "elasticsearch:9200:${ip_es}" ) @@ -87,7 +106,7 @@ EOD ) declare -a search_args=( '-s' '-u' 'elastic:testpasswd' - 'http://elasticsearch:9200/logs-generic-default/_search?pretty' + 'http://elasticsearch:9200/logstash-*/_search?pretty' '--resolve' "elasticsearch:9200:${ip_es}" '-H' 'Content-Type: application/json' '-d' "${query}" diff --git a/.github/workflows/scripts/run-tests-fleet.sh b/.github/workflows/scripts/run-tests-fleet.sh index 13dfae9..09c400a 100755 --- a/.github/workflows/scripts/run-tests-fleet.sh +++ b/.github/workflows/scripts/run-tests-fleet.sh @@ -107,72 +107,3 @@ if (( count == 0 )); then echo 'Expected at least 1 document' exit 1 fi - -log 'Searching a container document generated by Fleet Server' - -query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) < 0 )); then - break - fi - - was_retried=1 - echo -n 'x' >&2 - sleep 2 -done -if ((was_retried)); then - # flush stderr, important in non-interactive environments (CI) - echo >&2 -fi - -echo "$response" -if (( count == 0 )); then - echo 'Expected at least 1 document' - exit 1 -fi diff --git a/.github/workflows/spam-issue-close.yml b/.github/workflows/spam-issue-close.yml deleted file mode 100644 index 4de11c1..0000000 --- a/.github/workflows/spam-issue-close.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Close issues without context - -permissions: - issues: write - -on: - issues: - types: [ labeled ] - -jobs: - - close-lock: - name: Close and lock issues - if: contains(github.event.issue.labels.*.name, 'bot:close') && github.event.issue.state == 'open' - runs-on: ubuntu-latest - - steps: - - name: Close - id: close - uses: actions/stale@v9.0.0 - with: - days-before-issue-stale: -1 - days-before-issue-close: 0 - stale-issue-label: bot:close - close-issue-label: insufficient information - close-issue-message: >- - This description omits all, or critical parts of the information requested by maintainers to be able to - reproduce the issue: - - - - the **complete** log history of your Elastic components, including `setup`. - - any change(s) performed to the docker-elk configuration. - - details about the runtime environment, for both Docker and Compose. - - - Therefore, this issue will now be **closed**. Please open a new issue and fill in the template. It saves - everyone's efforts, and allows maintainers to provide you with a solution in as few round trips as possible. - - Thank you for your understanding. :pray: - - # Due to eventual consistency, listing closed issues immediately after a - # close does not always yield the expected results. A sleep is a simple - # enough remediation to this issue. - - name: Pause - if: fromJson(steps.close.outputs.closed-issues-prs)[0] - run: sleep 5 - - - name: Lock - uses: dessant/lock-threads@v5 - if: fromJson(steps.close.outputs.closed-issues-prs)[0] - with: - process-only: issues - issue-inactive-days: 0 - include-any-issue-labels: bot:close - remove-issue-labels: bot:close - issue-lock-reason: spam - log-output: true diff --git a/.github/workflows/update-merge.yml b/.github/workflows/update-merge.yml deleted file mode 100644 index b7670b1..0000000 --- a/.github/workflows/update-merge.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Merge Elastic updates - -on: - workflow_run: - workflows: [ CI ] - types: - - completed - branches: - - update/main - - update/tls - - update/release-7.x - -jobs: - - merge: - name: Merge pull request - if: github.event.workflow_run.conclusion == 'success' - runs-on: ubuntu-latest - - steps: - - name: Impersonate update bot - uses: actions/create-github-app-token@v1 - id: generate-token - with: - app-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - - - name: Approve and merge - uses: ridedott/merge-me-action@v2 - with: - GITHUB_LOGIN: docker-elk-updater - GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} - - - name: Delete branch - uses: actions/github-script@v7 - with: - script: | - await github.request('DELETE /repos/{owner}/{repo}/git/refs/{ref}', { - owner: '${{ github.event.workflow_run.repository.owner.login }}', - repo: '${{ github.event.workflow_run.repository.name }}', - ref: 'heads/${{ github.event.workflow_run.head_branch }}' - }) diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml deleted file mode 100644 index 09752da..0000000 --- a/.github/workflows/update.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Update Elastic release - -on: - schedule: - - cron: '0 0 * * 0' # At 00:00 every Sunday - -jobs: - - check-and-update: - name: Check and update Elastic release - runs-on: ubuntu-latest - strategy: - matrix: - include: - - release: 8.x - branch: main - - release: 8.x - branch: tls - - release: 7.x - branch: release-7.x - - steps: - - uses: actions/setup-node@v4 - - run: npm install semver - - - name: Get latest release version - uses: actions/github-script@v7 - id: get-latest-release - with: - script: | - const semver = require('semver') - - const latestVersion = await github. - paginate(github.rest.repos.listReleases, { - owner: 'elastic', - repo: 'elasticsearch' - }) - .then(releases => { - for (const release of releases) { - // Results are returned sorted by created_at, so it is safe to assume - // that the first encountered match is also the series' latest release. - - const version=semver.clean(release.tag_name) - - if (semver.satisfies(version, '${{ matrix.release }}')) { - return version - } - } - }); - - if (latestVersion) { - // Return an object so that the result can be handled as structured data - // instead of a quoted string in subsequent steps. - return { version: latestVersion } - } - - - uses: actions/checkout@v4 - if: steps.get-latest-release.outputs.result - with: - ref: ${{ matrix.branch }} - - - name: Update stack version - id: update-files - if: steps.get-latest-release.outputs.result - run: | - source .env - cur_ver="$ELASTIC_VERSION" - new_ver=${{ fromJson(steps.get-latest-release.outputs.result).version }} - - # Escape period characters so sed interprets them literally - cur_ver="${cur_ver//./\\.}" - - declare -a upd_files=( .env README.md */Dockerfile extensions/*/Dockerfile ) - if [ -f tls/README.md ]; then - upd_files+=( tls/README.md ) - fi - - sed -i "s/${cur_ver}/${new_ver}/g" "${upd_files[@]}" - - git_status="$(git status --porcelain)" - if [[ ${git_status} ]]; then - echo -e 'Changes to be committed:\n' - echo "${git_status}" - echo 'has-changes=true' >>"$GITHUB_OUTPUT" - fi - - - name: Impersonate update bot - uses: actions/create-github-app-token@v1 - id: generate-token - if: steps.update-files.outputs.has-changes - with: - app-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - - - name: Send pull request to update to new version - if: steps.update-files.outputs.has-changes - uses: peter-evans/create-pull-request@v7 - with: - token: ${{ steps.generate-token.outputs.token }} - branch: update/${{ matrix.branch }} - commit-message: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }} - title: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }} - delete-branch: true diff --git a/README.md b/README.md index b0dbf50..01f492a 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.15.2-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) -[![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.17.24-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=release-7.x)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Arelease-7.x) [![Join the chat](https://badges.gitter.im/Join%20Chat.svg)](https://app.gitter.im/#/room/#deviantony_docker-elk:gitter.im) -Run the latest version of the [Elastic stack][elk-stack] with Docker and Docker Compose. +Run the version 7.x of the [Elastic stack][elk-stack] with Docker and Docker Compose. It gives you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch and the visualization power of Kibana. Based on the [official Docker images][elastic-docker] from Elastic: -* [Elasticsearch](https://github.com/elastic/elasticsearch/tree/main/distribution/docker) -* [Logstash](https://github.com/elastic/logstash/tree/main/docker) -* [Kibana](https://github.com/elastic/kibana/tree/main/src/dev/build/tasks/os_packages/docker_generator) +* [Elasticsearch](https://github.com/elastic/elasticsearch/tree/7.17/distribution/docker) +* [Logstash](https://github.com/elastic/logstash/tree/7.17/docker) +* [Kibana](https://github.com/elastic/kibana/tree/7.17/src/dev/build/tasks/os_packages/docker_generator) Other available stack variants: @@ -70,6 +70,7 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p * [Initial setup](#initial-setup) * [Setting up user authentication](#setting-up-user-authentication) * [Injecting data](#injecting-data) + * [Default Kibana index pattern creation](#default-kibana-index-pattern-creation) * [Cleanup](#cleanup) * [Version selection](#version-selection) 1. [Configuration](#configuration) @@ -139,7 +140,7 @@ instructions from the [documentation][mac-filesharing] to add more locations. Clone this repository onto the Docker host that will run the stack with the command below: ```sh -git clone https://github.com/deviantony/docker-elk.git +git clone --branch release-7.x https://github.com/deviantony/docker-elk.git ``` Then, initialize the Elasticsearch users and groups required by docker-elk by executing the command: @@ -177,32 +178,20 @@ browser and use the following (default) credentials to log in: > [!NOTE] > Refer to [Security settings in Elasticsearch][es-security] to disable authentication. -> [!WARNING] -> Starting with Elastic v8.0.0, it is no longer possible to run Kibana using the bootstraped privileged `elastic` user. - The _"changeme"_ password set by default for all aforementioned users is **unsecure**. For increased security, we will reset the passwords of all aforementioned Elasticsearch users to random secrets. -1. Reset passwords for default users +1. Reset passwords for built-in users - The commands below reset the passwords of the `elastic`, `logstash_internal` and `kibana_system` users. Take note - of them. + The command below resets the passwords of all 6 [built-in users][builtin-users]. Take note of them. ```sh - docker compose exec elasticsearch bin/elasticsearch-reset-password --batch --user elastic + docker compose exec -T elasticsearch bin/elasticsearch-setup-passwords auto --batch ``` - ```sh - docker compose exec elasticsearch bin/elasticsearch-reset-password --batch --user logstash_internal - ``` - - ```sh - docker compose exec elasticsearch bin/elasticsearch-reset-password --batch --user kibana_system - ``` - - If the need for it arises (e.g. if you want to [collect monitoring information][ls-monitoring] through Beats and - other components), feel free to repeat this operation at any time for the rest of the [built-in - users][builtin-users]. + The password of the `logstash_internal` user must be changed manually, either from Kibana's [Stack Management > + Users][kibana-sec] view, or using the procedure described at [How to reset a password + programmatically](#how-to-reset-a-password-programmatically). 1. Replace usernames and passwords in configuration files @@ -256,6 +245,37 @@ cat /path/to/logfile.log | nc --send-only localhost 50000 # nmap You can also load the sample data provided by your Kibana installation. +#### Default Kibana index pattern creation + +When Kibana launches for the first time, it is not configured with any index pattern. + +##### Via the Kibana web UI + +> [!NOTE] +> You need to inject data into Logstash before being able to configure a Logstash index pattern via the Kibana web UI. + +Navigate to the _Discover_ view of Kibana from the left sidebar. You will be prompted to create an index pattern. Enter +`logstash-*` to match Logstash indices then, on the next page, select `@timestamp` as the time filter field. Finally, +click _Create index pattern_ and return to the _Discover_ view to inspect your log entries. + +Refer to [Connect Kibana with Elasticsearch][connect-kibana] and [Creating an index pattern][index-pattern] for detailed +instructions about the index pattern configuration. + +##### On the command line + +Create an index pattern via the Kibana API: + +```sh +curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ + -H 'Content-Type: application/json' \ + -H 'kbn-version: 7.17.24' \ + -u elastic: \ + -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' +``` + +The created pattern will automatically be marked as the default index pattern as soon as the Kibana UI is opened for the +first time. + ### Cleanup Elasticsearch data is persisted inside a volume by default. @@ -268,9 +288,6 @@ docker compose down -v ### Version selection -This repository stays aligned with the latest version of the Elastic stack. The `main` branch tracks the current major -version (8.x). - To use a different version of the core Elastic components, simply change the version number inside the [`.env`](.env) file. If you are upgrading an existing stack, remember to rebuild all container images using the `docker compose build` command. @@ -279,12 +296,6 @@ command. > Always pay attention to the [official upgrade instructions][upgrade] for each individual component before performing a > stack upgrade. -Older major versions are also supported on separate branches: - -* [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series -* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series (End-of-life) -* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-of-life) - ## Configuration > [!IMPORTANT] @@ -465,10 +476,10 @@ See the following Wiki pages: [elk-stack]: https://www.elastic.co/what-is/elk-stack [elastic-docker]: https://www.docker.elastic.co/ [subscriptions]: https://www.elastic.co/subscriptions -[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html -[license-settings]: https://www.elastic.co/guide/en/elasticsearch/reference/current/license-settings.html -[license-mngmt]: https://www.elastic.co/guide/en/kibana/current/managing-licenses.html -[license-apis]: https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html +[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/security-settings.html +[license-settings]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/license-settings.html +[license-mngmt]: https://www.elastic.co/guide/en/kibana/7.17/managing-licenses.html +[license-apis]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/licensing-apis.html [elastdocker]: https://github.com/sherifabdlnaby/elastdocker @@ -476,29 +487,29 @@ See the following Wiki pages: [compose-install]: https://docs.docker.com/compose/install/ [linux-postinstall]: https://docs.docker.com/engine/install/linux-postinstall/ -[bootstrap-checks]: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html -[es-sys-config]: https://www.elastic.co/guide/en/elasticsearch/reference/current/system-config.html -[es-heap]: https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings +[bootstap-checks]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/bootstrap-checks.html +[es-sys-config]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/system-config.html +[es-heap]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/important-settings.html#heap-size-settings [win-filesharing]: https://docs.docker.com/desktop/settings/windows/#file-sharing [mac-filesharing]: https://docs.docker.com/desktop/settings/mac/#file-sharing -[builtin-users]: https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html -[ls-monitoring]: https://www.elastic.co/guide/en/logstash/current/monitoring-with-metricbeat.html -[sec-cluster]: https://www.elastic.co/guide/en/elasticsearch/reference/current/secure-cluster.html +[builtin-users]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/built-in-users.html +[sec-cluster]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/secure-cluster.html +[kibana-sec]: https://www.elastic.co/guide/en/kibana/7.17/xpack-security.html -[connect-kibana]: https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html -[index-pattern]: https://www.elastic.co/guide/en/kibana/current/index-patterns.html +[connect-kibana]: https://www.elastic.co/guide/en/kibana/7.17/connect-to-elasticsearch.html +[index-pattern]: https://www.elastic.co/guide/en/kibana/7.17/index-patterns.html [config-es]: ./elasticsearch/config/elasticsearch.yml [config-kbn]: ./kibana/config/kibana.yml [config-ls]: ./logstash/config/logstash.yml -[es-docker]: https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html -[kbn-docker]: https://www.elastic.co/guide/en/kibana/current/docker.html -[ls-docker]: https://www.elastic.co/guide/en/logstash/current/docker-config.html +[es-docker]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/docker.html +[kbn-docker]: https://www.elastic.co/guide/en/kibana/7.17/docker.html +[ls-docker]: https://www.elastic.co/guide/en/logstash/7.17/docker-config.html -[upgrade]: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html +[upgrade]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/setup-upgrade.html