From 3574d36df98423f866946313f488b4c90b6cedb6 Mon Sep 17 00:00:00 2001 From: vivekgsharma Date: Fri, 22 Mar 2024 17:19:43 +0000 Subject: [PATCH 1/9] configured elastic_beat role to collect logs using metricbeat --- ansible/roles/elastic_beats/vars/tenderdash.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/roles/elastic_beats/vars/tenderdash.yml b/ansible/roles/elastic_beats/vars/tenderdash.yml index f958ec103..05bad9cee 100644 --- a/ansible/roles/elastic_beats/vars/tenderdash.yml +++ b/ansible/roles/elastic_beats/vars/tenderdash.yml @@ -22,3 +22,10 @@ platform_filebeat_inputs: fields: - from: "json.level" to: "log.level" + +metricbeat_modules: + - module: prometheus + period: 10s + hosts: ["localhost:{{ prometheus_port }}"] + metrics_path: "/metrics" + namespace: "drive_tenderdash" \ No newline at end of file From b18917097a06a1fe705df432f521f3638f359b80 Mon Sep 17 00:00:00 2001 From: vivekgsharma Date: Fri, 22 Mar 2024 17:37:40 +0000 Subject: [PATCH 2/9] small fix --- ansible/roles/elastic_beats/vars/tenderdash.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/roles/elastic_beats/vars/tenderdash.yml b/ansible/roles/elastic_beats/vars/tenderdash.yml index 05bad9cee..a29373cde 100644 --- a/ansible/roles/elastic_beats/vars/tenderdash.yml +++ b/ansible/roles/elastic_beats/vars/tenderdash.yml @@ -28,4 +28,5 @@ metricbeat_modules: period: 10s hosts: ["localhost:{{ prometheus_port }}"] metrics_path: "/metrics" - namespace: "drive_tenderdash" \ No newline at end of file + namespace: "drive_tenderdash" + \ No newline at end of file From 6c0d6a6547dfd5156a365750b17a06475f90ec74 Mon Sep 17 00:00:00 2001 From: vivekgsharma Date: Fri, 22 Mar 2024 18:11:18 +0000 Subject: [PATCH 3/9] added logs parser and alerts --- .../roles/elastic_beats/vars/tenderdash.yml | 3 +- .../pipelines/process_proposal_pipeline.json | 11 +++++ .../files/watches/error_logs_watch.json | 33 +++++++++++++ .../files/watches/high_round_watch.json | 41 +++++++++++++++++ .../files/watches/no_logs_watch.json | 46 +++++++++++++++++++ ansible/roles/elastic_stack/tasks/main.yml | 28 +++++++++++ 6 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/elastic_stack/files/pipelines/process_proposal_pipeline.json create mode 100644 ansible/roles/elastic_stack/files/watches/error_logs_watch.json create mode 100644 ansible/roles/elastic_stack/files/watches/high_round_watch.json create mode 100644 ansible/roles/elastic_stack/files/watches/no_logs_watch.json diff --git a/ansible/roles/elastic_beats/vars/tenderdash.yml b/ansible/roles/elastic_beats/vars/tenderdash.yml index a29373cde..3010b396b 100644 --- a/ansible/roles/elastic_beats/vars/tenderdash.yml +++ b/ansible/roles/elastic_beats/vars/tenderdash.yml @@ -28,5 +28,4 @@ metricbeat_modules: period: 10s hosts: ["localhost:{{ prometheus_port }}"] metrics_path: "/metrics" - namespace: "drive_tenderdash" - \ No newline at end of file + namespace: "drive_tenderdash" diff --git a/ansible/roles/elastic_stack/files/pipelines/process_proposal_pipeline.json b/ansible/roles/elastic_stack/files/pipelines/process_proposal_pipeline.json new file mode 100644 index 000000000..7a0261f0d --- /dev/null +++ b/ansible/roles/elastic_stack/files/pipelines/process_proposal_pipeline.json @@ -0,0 +1,11 @@ +{ + "description": "Parse round number from Tenderdash logs", + "processors": [ + { + "grok": { + "field": "message", + "patterns": ["Processed proposal with .* for height: %{NUMBER:height}, round: %{NUMBER:round}"] + } + } + ] + } \ No newline at end of file diff --git a/ansible/roles/elastic_stack/files/watches/error_logs_watch.json b/ansible/roles/elastic_stack/files/watches/error_logs_watch.json new file mode 100644 index 000000000..ac47c449d --- /dev/null +++ b/ansible/roles/elastic_stack/files/watches/error_logs_watch.json @@ -0,0 +1,33 @@ +{ + "trigger": { + "schedule": { + "interval": "10s" + } + }, + "input": { + "search": { + "request": { + "indices": ["logs-*"], + "body": { + "query": { + "bool": { + "should": [ + { "match": { "log.level": "ERROR" }}, + { "match": { "log.level": "FATAL" }} + ], + "minimum_should_match": 1 + } + } + } + } + } + }, + "actions": { + "log_alert": { + "logging": { + "text": "Alert triggered: {{ctx.payload}}" + } + } + } + } + \ No newline at end of file diff --git a/ansible/roles/elastic_stack/files/watches/high_round_watch.json b/ansible/roles/elastic_stack/files/watches/high_round_watch.json new file mode 100644 index 000000000..d6c965080 --- /dev/null +++ b/ansible/roles/elastic_stack/files/watches/high_round_watch.json @@ -0,0 +1,41 @@ +{ + "trigger": { + "schedule": { + "interval": "10s" + } + }, + "input": { + "search": { + "request": { + "indices": ["logs-*"], + "body": { + "query": { + "bool": { + "must": [ + { + "match": { + "message": "Processed proposal" + } + }, + { + "range": { + "round": { + "gt": 5 + } + } + } + ] + } + } + } + } + } + }, + "actions": { + "log_alert": { + "logging": { + "text": "Alert triggered: {{ctx.payload}}" + } + } + } + } \ No newline at end of file diff --git a/ansible/roles/elastic_stack/files/watches/no_logs_watch.json b/ansible/roles/elastic_stack/files/watches/no_logs_watch.json new file mode 100644 index 000000000..c83d8c3b7 --- /dev/null +++ b/ansible/roles/elastic_stack/files/watches/no_logs_watch.json @@ -0,0 +1,46 @@ +{ + "trigger": { + "schedule": { + "interval": "5m" + } + }, + "input": { + "search": { + "request": { + "indices": ["logs-*"], + "body": { + "query": { + "bool": { + "must_not": { + "exists": { + "field": "message" + } + }, + "filter": { + "range": { + "@timestamp": { + "gte": "now-5m" + } + } + } + } + } + } + } + } + }, + "condition": { + "compare": { + "ctx.payload.hits.total": { + "eq": 0 + } + } + }, + "actions": { + "log_alert": { + "logging": { + "text": "Alert triggered: {{ctx.payload}}" + } + } + } + } \ No newline at end of file diff --git a/ansible/roles/elastic_stack/tasks/main.yml b/ansible/roles/elastic_stack/tasks/main.yml index 2a5b955a6..88324c8ba 100644 --- a/ansible/roles/elastic_stack/tasks/main.yml +++ b/ansible/roles/elastic_stack/tasks/main.yml @@ -119,3 +119,31 @@ ansible.builtin.import_tasks: configure_cluster.yml run_once: true delegate_to: '{{ play_hosts | first }}' + + +- name: Upload and apply Tenderdash log parsing pipeline + ansible.builtin.uri: + url: "{{ elasticsearch_url }}/logstash/_pipeline/process_proposal_pipeline" + method: PUT + body: "{{ lookup('file', 'files/pipelines/process_proposal_pipeline.json') }}" + body_format: json + user: "{{ elastic_username }}" + password: "{{ elastic_password }}" + when: inventory_hostname in groups['elasticsearch'] + +- name: Create Elasticsearch Watches + ansible.builtin.uri: + url: "{{ elasticsearch_url }}/_watcher/watch/{{ item.name }}" + method: PUT + body: "{{ lookup('file', item.file) }}" + body_format: json + user: "{{ elastic_username }}" + password: "{{ elastic_password }}" + loop: + - { name: 'error_logs_watch', file: 'files/watches/error_logs_watch.json' } + - { name: 'high_round_watch', file: 'files/watches/high_round_watch.json' } + - { name: 'no_logs_watch', file: 'files/watches/no_logs_watch.json' } + loop_control: + label: "{{ item.name }}" + when: inventory_hostname in groups['elasticsearch'] + \ No newline at end of file From 7a2a823a864c8830535d56ddbbbcdd5ab735ffaa Mon Sep 17 00:00:00 2001 From: vivekgsharma Date: Sun, 24 Mar 2024 20:54:12 +0000 Subject: [PATCH 4/9] small fix --- ansible/roles/elastic_stack/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/roles/elastic_stack/tasks/main.yml b/ansible/roles/elastic_stack/tasks/main.yml index 88324c8ba..bfce2e189 100644 --- a/ansible/roles/elastic_stack/tasks/main.yml +++ b/ansible/roles/elastic_stack/tasks/main.yml @@ -146,4 +146,3 @@ loop_control: label: "{{ item.name }}" when: inventory_hostname in groups['elasticsearch'] - \ No newline at end of file From e7f157b868908fd2a44d10f941ca7aa710409aea Mon Sep 17 00:00:00 2001 From: vivekgsharma Date: Sun, 24 Mar 2024 20:58:04 +0000 Subject: [PATCH 5/9] smaall fix2 --- ansible/roles/elastic_beats/vars/tenderdash.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/elastic_beats/vars/tenderdash.yml b/ansible/roles/elastic_beats/vars/tenderdash.yml index 3010b396b..307e06748 100644 --- a/ansible/roles/elastic_beats/vars/tenderdash.yml +++ b/ansible/roles/elastic_beats/vars/tenderdash.yml @@ -26,6 +26,6 @@ platform_filebeat_inputs: metricbeat_modules: - module: prometheus period: 10s - hosts: ["localhost:{{ prometheus_port }}"] + hosts: ["localhost:{{ prometheus_port }}"] metrics_path: "/metrics" namespace: "drive_tenderdash" From 6a07011468c6b08980840a5626c7b25b44aa2750 Mon Sep 17 00:00:00 2001 From: vivekgsharma Date: Tue, 26 Mar 2024 11:43:20 +0000 Subject: [PATCH 6/9] fixed main.yml in elastic_beats --- ansible/roles/elastic_beats/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ansible/roles/elastic_beats/tasks/main.yml b/ansible/roles/elastic_beats/tasks/main.yml index 2646f6e0c..1b7480fb0 100644 --- a/ansible/roles/elastic_beats/tasks/main.yml +++ b/ansible/roles/elastic_beats/tasks/main.yml @@ -110,6 +110,12 @@ hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true + + - module: prometheus + period: 10s + hosts: ["{{private_ip}}:{{ prometheus_port }}"] + metrics_path: "/metrics" + namespace: "drive_tenderdash" output_conf: elasticsearch: hosts: From df00232becb6cb8d8b4930ee4b5ac6d8141ffb9d Mon Sep 17 00:00:00 2001 From: vivekgsharma Date: Tue, 26 Mar 2024 11:47:26 +0000 Subject: [PATCH 7/9] small fix --- ansible/roles/elastic_beats/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/elastic_beats/tasks/main.yml b/ansible/roles/elastic_beats/tasks/main.yml index 1b7480fb0..da950f2f9 100644 --- a/ansible/roles/elastic_beats/tasks/main.yml +++ b/ansible/roles/elastic_beats/tasks/main.yml @@ -113,7 +113,7 @@ - module: prometheus period: 10s - hosts: ["{{private_ip}}:{{ prometheus_port }}"] + hosts: ["{{ private_ip }}:{{ prometheus_port }}"] metrics_path: "/metrics" namespace: "drive_tenderdash" output_conf: From 25f7d21e5abe823125f1a4a16f1259d9117cc7d2 Mon Sep 17 00:00:00 2001 From: vivekgsharma Date: Tue, 26 Mar 2024 11:51:16 +0000 Subject: [PATCH 8/9] fix5 --- ansible/roles/elastic_beats/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/elastic_beats/tasks/main.yml b/ansible/roles/elastic_beats/tasks/main.yml index da950f2f9..98dba7598 100644 --- a/ansible/roles/elastic_beats/tasks/main.yml +++ b/ansible/roles/elastic_beats/tasks/main.yml @@ -110,7 +110,7 @@ hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true - + - module: prometheus period: 10s hosts: ["{{ private_ip }}:{{ prometheus_port }}"] From 6608e0715ad9531f61f3602ea035c53e98628a75 Mon Sep 17 00:00:00 2001 From: vivekgsharma Date: Tue, 26 Mar 2024 13:18:09 +0000 Subject: [PATCH 9/9] defined elasticsearch_uri --- ansible/roles/elastic_beats/vars/tenderdash.yml | 2 +- ansible/roles/elastic_stack/vars/main.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/roles/elastic_beats/vars/tenderdash.yml b/ansible/roles/elastic_beats/vars/tenderdash.yml index 307e06748..003fa7eaa 100644 --- a/ansible/roles/elastic_beats/vars/tenderdash.yml +++ b/ansible/roles/elastic_beats/vars/tenderdash.yml @@ -26,6 +26,6 @@ platform_filebeat_inputs: metricbeat_modules: - module: prometheus period: 10s - hosts: ["localhost:{{ prometheus_port }}"] + hosts: ["{{ private_ip }}:{{ prometheus_port }}"] metrics_path: "/metrics" namespace: "drive_tenderdash" diff --git a/ansible/roles/elastic_stack/vars/main.yml b/ansible/roles/elastic_stack/vars/main.yml index f99a9d3e3..db525ba30 100644 --- a/ansible/roles/elastic_stack/vars/main.yml +++ b/ansible/roles/elastic_stack/vars/main.yml @@ -4,3 +4,4 @@ bundle_path: '{{ elastic_path }}/bundle' certs_path: '{{ elastic_path }}/certificates' data_path: '{{ elastic_path }}/data' kibana_port: 5601 +elasticsearch_url: "http://{{ hostvars[inventory_hostname].private_ip }}:9200"