From 32e16fd7ae3819aa0f8301eccd920ca7aa3020f5 Mon Sep 17 00:00:00 2001 From: aasdhajkshd <71592286+aasdhajkshd@users.noreply.github.com> Date: Sat, 18 Nov 2023 14:28:00 +0300 Subject: [PATCH 1/7] logging-1 --- docker/.env.example | 12 ++-- docker/.gitignore | 1 + docker/docker-compose-apps.yml | 101 ++++++++++++++++++++++++++++++ docker/docker-compose-logging.yml | 25 ++++++++ docker/docker-compose.yml | 20 +++++- 5 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 docker/docker-compose-apps.yml create mode 100644 docker/docker-compose-logging.yml diff --git a/docker/.env.example b/docker/.env.example index e14d824..c739b09 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1,6 +1,10 @@ USERNAME=23f03013e37f -PORT=8080 -UI_VERSION=1.0 -POST_VERSION=1.0 -COMMENT_VERSION=1.0 +PORT=9292 +UI_VERSION=latest +POST_VERSION=logging +COMMENT_VERSION=logging DB_VERSION=4.4.24 +PROMETHEUS_VERSION=2.1.0 +FLUENT_VERSION=logging +COMPOSE_PROJECT_NAME=logging +ZIPKIN_ENABLED=true diff --git a/docker/.gitignore b/docker/.gitignore index 76a6230..d6450c5 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,3 +1,4 @@ +.secrets/* **/.env build_info.txt *.pyc diff --git a/docker/docker-compose-apps.yml b/docker/docker-compose-apps.yml new file mode 100644 index 0000000..359ea93 --- /dev/null +++ b/docker/docker-compose-apps.yml @@ -0,0 +1,101 @@ +version: '3.3' +name: logging +services: + post_db: + image: mongo:${DB_VERSION} + volumes: + - post_db:/data/db + networks: + - back_net + ui: + image: ${USERNAME}/ui:${UI_VERSION} + environment: + - ZIPKIN_ENABLED=${ZIPKIN_ENABLED} + ports: + - ${PORT}:9292/tcp + networks: + - front_net + depends_on: + - fluent-bit + - post_db + - post + - comment + command: puma --debug -w 2 + logging: + driver: "fluentd" + options: + fluentd-address: localhost:24224 + tag: service.ui + post: + image: ${USERNAME}/post:${POST_VERSION} + environment: + - ZIPKIN_ENABLED=${ZIPKIN_ENABLED} + - COMPOSE_PROJECT_NAME + networks: + - back_net + - front_net + logging: + driver: "fluentd" + options: + fluentd-address: localhost:24224 + tag: service.post + depends_on: + - fluent-bit + comment: + image: ${USERNAME}/comment:${COMMENT_VERSION} + environment: + - ZIPKIN_ENABLED=${ZIPKIN_ENABLED} + logging: + driver: "fluentd" + options: + fluentd-address: localhost:24224 + tag: service.post + networks: + - back_net + - front_net + depends_on: + - fluent-bit + zipkin: + image: openzipkin/zipkin:2.21.0 + ports: + - "9411:9411" + networks: + - back_net + - front_net + fluent-bit: + image: ${USERNAME}/fluent-bit:${FLUENT_VERSION} + ports: + - "24224:24224" + - "24224:24224/udp" + depends_on: + - elasticsearch + networks: + - back_net + elasticsearch: + image: elasticsearch:7.17.3 + environment: + - discovery.type=single-node + expose: + - "9200" + ports: + - "9200:9200" + networks: + - back_net + kibana: + image: kibana:7.17.3 + ports: + - "5601:5601" + networks: + - front_net +volumes: + post_db: + prometheus_data: +networks: + front_net: + ipam: + config: + - subnet: 10.0.1.0/24 + back_net: + ipam: + config: + - subnet: 10.0.2.0/24 diff --git a/docker/docker-compose-logging.yml b/docker/docker-compose-logging.yml new file mode 100644 index 0000000..0b0ea17 --- /dev/null +++ b/docker/docker-compose-logging.yml @@ -0,0 +1,25 @@ +version: '3' +services: + zipkin: + image: openzipkin/zipkin:2.21.0 + ports: + - "9411:9411" + fluent-bit: + image: ${USERNAME}/fluent-bit:${FLUENT_VERSION} + ports: + - "24224:24224" + - "24224:24224/udp" + depends_on: + - elasticsearch + elasticsearch: + image: elasticsearch:7.17.3 + environment: + - discovery.type=single-node + expose: + - "9200" + ports: + - "9200:9200" + kibana: + image: kibana:7.17.3 + ports: + - "5601:5601" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0f4ff01..927963c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,4 +1,5 @@ version: '3.3' +name: logging services: post_db: image: mongo:${DB_VERSION} @@ -8,6 +9,8 @@ services: - back_net ui: image: ${USERNAME}/ui:${UI_VERSION} + environment: + - ZIPKIN_ENABLED=${ZIPKIN_ENABLED} ports: - ${PORT}:9292/tcp networks: @@ -17,13 +20,28 @@ services: - post - comment command: puma --debug -w 2 + logging: + driver: "fluentd" + options: + fluentd-address: localhost:24224 + tag: service.ui post: image: ${USERNAME}/post:${POST_VERSION} + environment: + - ZIPKIN_ENABLED=${ZIPKIN_ENABLED} + - COMPOSE_PROJECT_NAME networks: - back_net - front_net + logging: + driver: "fluentd" + options: + fluentd-address: localhost:24224 + tag: service.post comment: image: ${USERNAME}/comment:${COMMENT_VERSION} + environment: + - ZIPKIN_ENABLED=${ZIPKIN_ENABLED} networks: - back_net - front_net @@ -73,11 +91,9 @@ services: networks: - back_net - front_net - volumes: post_db: prometheus_data: - networks: front_net: ipam: From ef5bac7767b18ced2814209a1367727b33a9a7df Mon Sep 17 00:00:00 2001 From: aasdhajkshd <71592286+aasdhajkshd@users.noreply.github.com> Date: Sat, 18 Nov 2023 14:38:41 +0300 Subject: [PATCH 2/7] logging-1 --- README.md | 365 ++++++++++ bugged-code/.env | 10 + bugged-code/.gitignore | 7 + bugged-code/README.md | 1 + bugged-code/comment/Dockerfile | 17 + bugged-code/comment/Gemfile | 9 + bugged-code/comment/Gemfile.lock | 43 ++ bugged-code/comment/VERSION | 1 + bugged-code/comment/comment_app.rb | 123 ++++ bugged-code/comment/config.ru | 10 + bugged-code/comment/docker_build.sh | 6 + bugged-code/comment/helpers.rb | 75 ++ bugged-code/docker-compose-apps.yml | 101 +++ bugged-code/post-py/Dockerfile | 15 + bugged-code/post-py/VERSION | 1 + bugged-code/post-py/docker_build.sh | 6 + bugged-code/post-py/helpers.py | 39 + bugged-code/post-py/post_app.py | 247 +++++++ bugged-code/post-py/requirements.txt | 6 + bugged-code/ui/Dockerfile | 19 + bugged-code/ui/Gemfile | 13 + bugged-code/ui/Gemfile.lock | 75 ++ bugged-code/ui/VERSION | 1 + bugged-code/ui/config.ru | 26 + bugged-code/ui/docker_build.sh | 6 + bugged-code/ui/helpers.rb | 82 +++ bugged-code/ui/middleware.rb | 32 + bugged-code/ui/ui_app.rb | 203 ++++++ bugged-code/ui/views/create.haml | 10 + bugged-code/ui/views/index.haml | 27 + bugged-code/ui/views/layout.haml | 35 + bugged-code/ui/views/show.haml | 49 ++ img/Screenshot_20231117_223719.png | Bin 0 -> 325118 bytes img/Screenshot_20231117_225018.png | Bin 0 -> 235028 bytes img/Screenshot_20231118_095027.png | Bin 0 -> 91388 bytes img/Screenshot_20231118_095423.png | Bin 0 -> 424320 bytes img/Screenshot_20231118_100825.png | Bin 0 -> 331479 bytes img/Screenshot_20231118_101002.png | Bin 0 -> 342474 bytes img/Screenshot_20231118_101218.png | Bin 0 -> 319803 bytes img/Screenshot_20231118_103703.png | Bin 0 -> 212012 bytes img/Screenshot_20231118_104112.png | Bin 0 -> 62627 bytes img/Screenshot_20231118_113217.png | Bin 0 -> 52060 bytes img/Screenshot_20231118_115610.png | Bin 0 -> 56823 bytes img/Screenshot_20231118_124003.png | Bin 0 -> 42043 bytes img/Screenshot_20231118_124132.png | Bin 0 -> 100217 bytes img/Screenshot_20231118_135215.png | Bin 0 -> 79350 bytes img/Screenshot_20231118_135339.png | Bin 0 -> 17906 bytes img/Screenshot_20231118_142007.png | Bin 0 -> 19053 bytes logging/.gitignore | 1 + logging/README.md | 687 ++++++++++++++++++ logging/fluent-bit/Dockerfile | 3 + logging/fluent-bit/fluent-bit.conf | 51 ++ logging/fluent-bit/parsers.conf | 9 + logging/fluentd/Dockerfile | 6 + logging/fluentd/fluent.conf | 39 + logging/infra/.gitignore | 1 + logging/infra/ansible/.ansible-lint | 23 + logging/infra/ansible/.gitignore | 1 + logging/infra/ansible/ansible.cfg | 15 + .../ansible/environments/stage/group_vars/all | 1 + .../playbooks/docker-machine_install.yml | 8 + .../ansible/playbooks/docker_install.yml | 8 + .../ansible/playbooks/docker_logging.yml | 178 +++++ logging/infra/ansible/requirements.txt | 1 + .../ansible/roles/docker-machine/README.md | 38 + .../roles/docker-machine/defaults/main.yml | 3 + .../roles/docker-machine/handlers/main.yml | 18 + .../roles/docker-machine/meta/main.yml | 9 + .../roles/docker-machine/tasks/main.yml | 143 ++++ .../roles/docker-machine/tests/inventory | 1 + .../roles/docker-machine/tests/test.yml | 5 + .../roles/docker-machine/vars/main.yml | 5 + logging/infra/ansible/roles/docker/README.md | 38 + .../ansible/roles/docker/defaults/main.yml | 3 + .../ansible/roles/docker/handlers/main.yml | 9 + .../infra/ansible/roles/docker/meta/main.yml | 9 + .../infra/ansible/roles/docker/tasks/main.yml | 90 +++ .../ansible/roles/docker/tests/inventory | 1 + .../infra/ansible/roles/docker/tests/test.yml | 5 + .../infra/ansible/roles/docker/vars/main.yml | 4 + logging/infra/terraform/.gitignore | 1 + .../infra/terraform/modules/docker/.gitignore | 1 + .../infra/terraform/modules/docker/main.tf | 98 +++ .../infra/terraform/modules/docker/outputs.tf | 19 + .../terraform/modules/docker/variables.tf | 89 +++ logging/infra/terraform/stage/.gitignore | 1 + logging/infra/terraform/stage/.tflint.hcl | 13 + logging/infra/terraform/stage/main.tf | 17 + logging/infra/terraform/stage/meta.yaml | 13 + logging/infra/terraform/stage/outputs.tf | 24 + logging/infra/terraform/stage/resources.tf | 25 + .../stage/templates/inventory.json.tpl | 20 + logging/infra/terraform/stage/variables.tf | 83 +++ .../infra/terraform/terraform.tfvars.example | 13 + src/comment/docker_build.sh | 2 +- src/post-py/.gitlab-ci.yml | 215 ++++++ src/post-py/Dockerfile | 15 + src/post-py/VERSION | 1 + src/post-py/docker_build.sh | 6 + src/post-py/helpers.py | 39 + src/post-py/post_app.py | 276 +++++++ src/post-py/requirements.txt | 6 + src/ui/docker_build.sh | 2 +- 103 files changed, 4050 insertions(+), 2 deletions(-) create mode 100644 bugged-code/.env create mode 100644 bugged-code/.gitignore create mode 100644 bugged-code/README.md create mode 100644 bugged-code/comment/Dockerfile create mode 100644 bugged-code/comment/Gemfile create mode 100644 bugged-code/comment/Gemfile.lock create mode 100644 bugged-code/comment/VERSION create mode 100644 bugged-code/comment/comment_app.rb create mode 100644 bugged-code/comment/config.ru create mode 100644 bugged-code/comment/docker_build.sh create mode 100644 bugged-code/comment/helpers.rb create mode 100644 bugged-code/docker-compose-apps.yml create mode 100644 bugged-code/post-py/Dockerfile create mode 100644 bugged-code/post-py/VERSION create mode 100644 bugged-code/post-py/docker_build.sh create mode 100644 bugged-code/post-py/helpers.py create mode 100644 bugged-code/post-py/post_app.py create mode 100644 bugged-code/post-py/requirements.txt create mode 100644 bugged-code/ui/Dockerfile create mode 100644 bugged-code/ui/Gemfile create mode 100644 bugged-code/ui/Gemfile.lock create mode 100644 bugged-code/ui/VERSION create mode 100644 bugged-code/ui/config.ru create mode 100644 bugged-code/ui/docker_build.sh create mode 100644 bugged-code/ui/helpers.rb create mode 100644 bugged-code/ui/middleware.rb create mode 100644 bugged-code/ui/ui_app.rb create mode 100644 bugged-code/ui/views/create.haml create mode 100644 bugged-code/ui/views/index.haml create mode 100644 bugged-code/ui/views/layout.haml create mode 100644 bugged-code/ui/views/show.haml create mode 100644 img/Screenshot_20231117_223719.png create mode 100644 img/Screenshot_20231117_225018.png create mode 100644 img/Screenshot_20231118_095027.png create mode 100644 img/Screenshot_20231118_095423.png create mode 100644 img/Screenshot_20231118_100825.png create mode 100644 img/Screenshot_20231118_101002.png create mode 100644 img/Screenshot_20231118_101218.png create mode 100644 img/Screenshot_20231118_103703.png create mode 100644 img/Screenshot_20231118_104112.png create mode 100644 img/Screenshot_20231118_113217.png create mode 100644 img/Screenshot_20231118_115610.png create mode 100644 img/Screenshot_20231118_124003.png create mode 100644 img/Screenshot_20231118_124132.png create mode 100644 img/Screenshot_20231118_135215.png create mode 100644 img/Screenshot_20231118_135339.png create mode 100644 img/Screenshot_20231118_142007.png create mode 100644 logging/.gitignore create mode 100644 logging/README.md create mode 100644 logging/fluent-bit/Dockerfile create mode 100644 logging/fluent-bit/fluent-bit.conf create mode 100644 logging/fluent-bit/parsers.conf create mode 100644 logging/fluentd/Dockerfile create mode 100644 logging/fluentd/fluent.conf create mode 100644 logging/infra/.gitignore create mode 100644 logging/infra/ansible/.ansible-lint create mode 100644 logging/infra/ansible/.gitignore create mode 100644 logging/infra/ansible/ansible.cfg create mode 100644 logging/infra/ansible/environments/stage/group_vars/all create mode 100644 logging/infra/ansible/playbooks/docker-machine_install.yml create mode 100644 logging/infra/ansible/playbooks/docker_install.yml create mode 100644 logging/infra/ansible/playbooks/docker_logging.yml create mode 100644 logging/infra/ansible/requirements.txt create mode 100644 logging/infra/ansible/roles/docker-machine/README.md create mode 100644 logging/infra/ansible/roles/docker-machine/defaults/main.yml create mode 100644 logging/infra/ansible/roles/docker-machine/handlers/main.yml create mode 100644 logging/infra/ansible/roles/docker-machine/meta/main.yml create mode 100644 logging/infra/ansible/roles/docker-machine/tasks/main.yml create mode 100644 logging/infra/ansible/roles/docker-machine/tests/inventory create mode 100644 logging/infra/ansible/roles/docker-machine/tests/test.yml create mode 100644 logging/infra/ansible/roles/docker-machine/vars/main.yml create mode 100644 logging/infra/ansible/roles/docker/README.md create mode 100644 logging/infra/ansible/roles/docker/defaults/main.yml create mode 100644 logging/infra/ansible/roles/docker/handlers/main.yml create mode 100644 logging/infra/ansible/roles/docker/meta/main.yml create mode 100644 logging/infra/ansible/roles/docker/tasks/main.yml create mode 100644 logging/infra/ansible/roles/docker/tests/inventory create mode 100644 logging/infra/ansible/roles/docker/tests/test.yml create mode 100644 logging/infra/ansible/roles/docker/vars/main.yml create mode 100644 logging/infra/terraform/.gitignore create mode 100644 logging/infra/terraform/modules/docker/.gitignore create mode 100644 logging/infra/terraform/modules/docker/main.tf create mode 100644 logging/infra/terraform/modules/docker/outputs.tf create mode 100644 logging/infra/terraform/modules/docker/variables.tf create mode 100644 logging/infra/terraform/stage/.gitignore create mode 100644 logging/infra/terraform/stage/.tflint.hcl create mode 100644 logging/infra/terraform/stage/main.tf create mode 100644 logging/infra/terraform/stage/meta.yaml create mode 100644 logging/infra/terraform/stage/outputs.tf create mode 100644 logging/infra/terraform/stage/resources.tf create mode 100644 logging/infra/terraform/stage/templates/inventory.json.tpl create mode 100644 logging/infra/terraform/stage/variables.tf create mode 100644 logging/infra/terraform/terraform.tfvars.example create mode 100644 src/post-py/.gitlab-ci.yml create mode 100644 src/post-py/Dockerfile create mode 100644 src/post-py/VERSION create mode 100644 src/post-py/docker_build.sh create mode 100644 src/post-py/helpers.py create mode 100644 src/post-py/post_app.py create mode 100644 src/post-py/requirements.txt diff --git a/README.md b/README.md index 2f84750..bccfdc7 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,371 @@ aasdhajkshd microservices repository * [kubernetes-3 Kubernetes. Networks, Storages](#hw30) * [kubernetes-4 CI/CD в Kubernetes](#hw31) * [kubernetes-2 Kubernetes. Запуск кластера и приложения. Модель безопасности](#hw29) +* [logging-1 Логирование и распределенная трассировка](#hw25) + +--- + +## Логирование и распределенная трассировка + +#### Выполненные работы + +--- + +Подготовка окружения docker-хост с именем logging в Yandex.Cloud и настройка локального окружение на работу с ним + +```bash +YC_HOSTNAME="docker-host" + +yc compute instance create \ + --name ${YC_HOSTNAME} \ + --zone ru-central1-a \ + --core-fraction 50 \ + --cores 4 \ + --memory 4GB \ + --network-interface subnet-name=default-ru-central1-a,nat-ip-version=ipv4 \ + --create-boot-disk image-folder-id=standard-images,image-family=ubuntu-2004-lts,size=40,type=network-ssd \ + --ssh-key ~/.ssh/id_rsa-appuser.pub \ + | awk '/nat:/ { getline; print $2}' + +YC_HOST_IP=$(yc compute instance list --format json | jq ".[] | select (.name == \"${YC_HOSTNAME}\") | .network_interfaces[0].primary_v4_address.one_to_one_nat.address" | tr -d '"') + +cat << EOF > docker/ansible/environment/stage/inventory.json +{ + "all": { + "hosts": { + "docker-host" : { + "ansible_host": "$YC_HOST_IP" + } + } + } +} +EOF + +# docker-machine rm -f ${YC_HOSTNAME} + +docker-machine create \ + --driver generic \ + --generic-ip-address=$YC_HOST_IP \ + --generic-ssh-user yc-user \ + --generic-ssh-key ~/.ssh/id_rsa-appuser \ + ${YC_HOSTNAME} + +docker-machine ls +eval $(docker-machine env ${YC_HOSTNAME}) +docker-machine ssh ${YC_HOSTNAME} + +sudo add-apt-repository ppa:longsleep/golang-backports +sudo apt update +sudo apt install golang-go + +go mod init github.com/yandex-cloud/docker-machine-driver-yandex +go install github.com/yandex-cloud/docker-machine-driver-yandex@latest + +curl -L https://github.com/docker/machine/releases/download/v0.8.2/docker-machine-`uname -s`-`uname -m` >/usr/local/bin/docker-machine && \ +chmod +x /usr/local/bin/docker-machine + +export PATH="$PATH:$HOME/go/bin" + +``` + +Создадим сервисную учетную запись + +```bash +export YC_FOLDER_ID='b1g0da3u1gqk0nansi59' +export SA_KEY_PATH="$HOME/key.json" + +yc iam service-account create --name=sa-default --folder-id=$YC_FOLDER_ID +yc iam key create --service-account-name sa-default --output $SA_KEY_PATH --folder-id $YC_FOLDER_ID +yc resource-manager folder add-access-binding \ + --name=infra \ + --id=$YC_FOLDER_ID \ + --service-account-id=$(yc iam service-account get sa-default | grep ^id | awk '{print $2}') \ + --role=editor +``` + +В папке подготовил **docker/ansbile** ansible playbook для установки docker-machine `ansible-playbook playbooks/docker-machine_install.yml` и установку go *Yandex.Cloud Docker machine driver* + +```bash +docker-machine -D create \ +--driver yandex \ +--yandex-image-family "ubuntu-2004-lts" \ +--yandex-platform-id "standard-v1" \ +--yandex-folder-id $YC_FOLDER_ID \ +--yandex-sa-key-file $SA_KEY_PATH \ +--yandex-memory "4" \ +--yandex-nat=true \ +logging + +eval $(docker-machine env logging) +docker-machine ip logging +``` + +Установка завершается ошибкой, разбираться нет смысла, так как docker-machine не является актуальным и более не поддерживается [Deprecate Docker Machine](https://github.com/docker/roadmap/issues/245). + +> Результат: + +```output +Error creating machine: Error running provisioning: Something went wrong running an SSH command! +command : DEBIAN_FRONTEND=noninteractive sudo -E apt-get install -y curl +err : exit status 100 +output : E: Could not get lock /var/lib/dpkg/lock-frontend - open (11: Resource temporarily unavailable) +E: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), is another process using it? + +... +Error running SSH command: Something went wrong running an SSH command! +command : netstat -tln +err : exit status 127 +output : bash: netstat: command not found + + +Error creating machine: Error running provisioning: Unable to verify the Docker daemon is listening: Maximum number of retries (10) exceeded +``` + +Дальнейшее разворачивание было выполнено через `terraform` и `ansible`. Файлы установки располагаются в папке `logging/infra/terraform` и `logging/infra/ansible`. Terraform `terraform -chdir=logging/infra/terraform/stage apply` создает машины с именами *docker-host-X* и *docker-logging-X*, запускает `logging/infra/ansible/playbooks/docker_logging.yml`, где выполняется установка docker-compose и запуск контейнеров. Причём адрес fluent сервера указывается изначально при установке из динамического `inventory.json` файла. + +### Логирование Docker-контейнеров + +[Dockerfile для fluentd](https://docs.fluentd.org/container-deployment/docker-compose) в итоге с различными версиями не взлетел с плагином *fluent-plugin-elasticsearch* + +```bash +mkdir -p logging/fluentd + +cat << EOF > logging/fluentd/Dockerfile +FROM fluent/fluentd:v0.12 +RUN gem install faraday-net_http -v 2.1.0 +RUN gem install faraday -v 1.10.3 +RUN gem install fluent-plugin-elasticsearch --no-rdoc --no-ri --version 1.9.5 +RUN gem install fluent-plugin-grok-parser --no-rdoc --no-ri --version 1.0.0 +COPY --chmod=644 fluent.conf /fluentd/etc +EOF +``` + +> Результат: +> +```output +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 [warn]: temporarily failed to flush the buffer. next_retry=2023-11-17 21:07:32 +0000 error_class="NameError" error="uninitialized constant Elasticsearch::Transport" plugin_id="object:2abd066abb50" +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 [warn]: /usr/lib/ruby/gems/2.5.0/gems/fluent-plugin-elasticsearch-1.9.5/lib/fluent/plugin/out_elasticsearch.rb:147:in `client' +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 [warn]: /usr/lib/ruby/gems/2.5.0/gems/fluent-plugin-elasticsearch-1.9.5/lib/fluent/plugin/out_elasticsearch.rb:359:in `rescue in send_bulk' +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 [warn]: /usr/lib/ruby/gems/2.5.0/gems/fluent-plugin-elasticsearch-1.9.5/lib/fluent/plugin/out_elasticsearch.rb:354:in `send_bulk' +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 [warn]: /usr/lib/ruby/gems/2.5.0/gems/fluent-plugin-elasticsearch-1.9.5/lib/fluent/plugin/out_elasticsearch.rb:341:in `write_objects' +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 [warn]: /usr/lib/ruby/gems/2.5.0/gems/fluentd-0.12.43/lib/fluent/output.rb:490:in `write' +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 [warn]: /usr/lib/ruby/gems/2.5.0/gems/fluentd-0.12.43/lib/fluent/buffer.rb:354:in `write_chunk' +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 [warn]: /usr/lib/ruby/gems/2.5.0/gems/fluentd-0.12.43/lib/fluent/buffer.rb:333:in `pop' +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 [warn]: /usr/lib/ruby/gems/2.5.0/gems/fluentd-0.12.43/lib/fluent/output.rb:342:in `try_flush' +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 [warn]: /usr/lib/ruby/gems/2.5.0/gems/fluentd-0.12.43/lib/fluent/output.rb:149:in `run' +logging-fluentd-1 | 2023-11-17 21:07:32 +0000 fluent.warn: {"next_retry":"2023-11-17 21:07:32 +0000","error_class":"NameError","error":"uninitialized constant Elasticsearch::Transport","plugin_id":"object:2abd066abb50","message":"temporarily failed to flush the buffer. next_retry=2023-11-17 21:07:32 +0000 error_class=\"NameError\" error=\"uninitialized constant Elasticsearch::Transport\" plugin_id=\"object:2abd066abb50\""} +``` + +Принято решение отказаться от fluentd и использовать [fluent-bit](https://docs.fluentbit.io/manual/pipeline/outputs/elasticsearch) +Подсмотрев установку в keubernetes, использовал эти же версии [Kubernetes Observability: логгинг с EFK](https://habr.com/ru/companies/otus/articles/721004/) из блога **Блог компании OTUS, Kubernetes** + +В итоге: + +```bash + +mkdir -p logging/fluent-bit + +cat << EOF > logging/fluent-bit/fluent-bit.conf +[INPUT] + name elasticsearch + listen 0.0.0.0 + port 9200 +[OUTPUT] + name stdout + match * +[OUTPUT] + Name es + Match service.* + Host elasticsearch + Logstash_Format On + Logstash_Prefix fluentd + Logstash_Dateformat %Y%m%d + Include_Tag_Key true + Retry_Limit False + tls Off + tls.verify Off + HTTP_User elastic + HTTP_Passwd elastic + Suppress_Type_Name On + Index fluentbit + Type docker +[INPUT] + Name forward + Listen 0.0.0.0 + Port 24224 + Buffer_Chunk_Size 1M + Buffer_Max_Size 6M +[SERVICE] + Flush 5 + Daemon Off + Log_Level debug +EOF + +cat << EOF > logging/fluent-bit/Dockerfile +FROM cr.fluentbit.io/fluent/fluent-bit +COPY --chmod=644 fluent-bit.conf /fluent-bit/etc/fluent-bit.conf +EOF + +docker buildx build --push -t ${USERNAME}/fluent-bit:logging . + +cat << EOF > docker/docker-compose-logging.yml +version: '3' +services: + fluent-bit: + image: ${USERNAME}/fluent-bit:${FLUENTD_VERSION} + ports: + - "24224:24224" + - "24224:24224/udp" + depends_on: + - elasticsearch + elasticsearch: + image: elasticsearch:7.17.3 + environment: + - discovery.type=single-node + expose: + - "9200" + ports: + - "9200:9200" + kibana: + image: kibana:7.17.3 + ports: + - "5601:5601" +EOF +``` + +Выполнена пересборка образов ui, comment, post c tag'ом - logging и доустановленными пакетами gcc и musl-dev + +```bash +for i in post ui comment; do cd $i && docker buildx build --push -t $USERNAME/$i:logging . && cd - ; done +``` + +> Результат см. `logging/README.md`: +> +```output +Apply complete! Resources: 4 added, 0 changed, 0 destroyed. + +Outputs: + +docker_host_instance = [ + [ + "fhma1onfpi80lu9p769r", + ], + [ + "docker-host-0", + ], +] +docker_host_ip_address = "158.160.125.139" +docker_image_id = "fd853sqaosrb2anl1uve" +docker_logging_instance = [ + [ + "fhmm27372bjc4i5elva1", + ], + [ + "docker-logging-0", + ], +] +docker_logging_ip_address = "158.160.106.143" + +ubuntu@fhma1onfpi80lu9p769r:~/docker$ docker-compose ps +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +logging-comment-1 23f03013e37f/comment:logging "puma" comment 7 minutes ago Up 7 minutes +logging-post-1 23f03013e37f/post:logging "python3 post_app.py" post 7 minutes ago Up 7 minutes +logging-post_db-1 mongo:4.4.24 "docker-entrypoint.s…" post_db 7 minutes ago Up 7 minutes 27017/tcp +logging-ui-1 23f03013e37f/ui:latest "puma --debug -w 2" ui 7 minutes ago Up 7 minutes 0.0.0.0:9292->9292/tcp, :::9292->9292/tcp + +ubuntu@fhmm27372bjc4i5elva1:~/docker$ docker-compose ps +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +logging-elasticsearch-1 elasticsearch:7.17.3 "/bin/tini -- /usr/l…" elasticsearch 9 minutes ago Up 9 minutes 0.0.0.0:9200->9200/tcp, :::9200->9200/tcp, 9300/tcp +logging-fluentd-1 23f03013e37f/fluent-bit:logging "/fluent-bit/bin/flu…" fluentd 9 minutes ago Up 9 minutes 2020/tcp, 0.0.0.0:24224->24224/tcp, 0.0.0.0:24224->24224/udp, :::24224->24224/tcp, :::24224->24224/udp +logging-kibana-1 kibana:7.17.3 "/bin/tini -- /usr/l…" kibana 9 minutes ago Up 9 minutes 0.0.0.0:5601->5601/tcp, :::5601->5601/tcp +``` + +### Cбор структурированных логов + + +![Reference](img/Screenshot_20231118_100825.png) +![Reference](img/Screenshot_20231118_101218.png) + +### Визуализация + +![Reference](img/Screenshot_20231118_103703.png) + +### Сбор неструктурированных логов + +![Reference](img/Screenshot_20231118_104112.png) + +```bash +mkdir -p logging/fluent-bit +cat << EOF >> logging/fluent-bit/fluent-bit.conf +[SERVICE] + flush 1 + log_level info + parsers_file parsers.conf +[FILTER] + Name parser + Match service.post + Key_Name log + Parser json_parser +[FILTER] + Name parser + Match service.ui + Key_Name log + Parser ui_parser +EOF +cat << EOF >> logging/fluent-bit/parsers.conf +[PARSER] + Name json_parser + Format json + Key_Name log +[PARSER] + Name ui_parser + Format regex + Regex /\[(?