From afddde9db4525e9797fd48fdf81cbc66ea529fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 4 Mar 2024 14:17:36 +0100 Subject: [PATCH 1/5] Add grafana cloud remote writer --- .../host_vars/monitoring.ooni.org/vars.yml | 1 + ansible/host_vars/monitoring.ooni.org/vault | 59 +++++++++++-------- ansible/roles/prometheus/defaults/main.yml | 1 + .../prometheus/templates/prometheus.yml.j2 | 5 ++ 4 files changed, 41 insertions(+), 25 deletions(-) diff --git a/ansible/host_vars/monitoring.ooni.org/vars.yml b/ansible/host_vars/monitoring.ooni.org/vars.yml index 9dd0a898c..1f9dcfa41 100644 --- a/ansible/host_vars/monitoring.ooni.org/vars.yml +++ b/ansible/host_vars/monitoring.ooni.org/vars.yml @@ -13,6 +13,7 @@ prometheus_basic_auth_password: "{{ vault_prometheus_basic_auth_password }}" alertmanager_slack_api_url: '{{ vault_alertmanager_slack_api_url }}' grafana_admin_password: "{{ vault_grafana_admin_password }}" grafana_secret_key: "{{ vault_grafana_secret_key }}" # to sign cookies +grafana_cloud_api_token: "{{ vault_grafana_cloud_api_token }}" # for sending data to grafana cloud prometheus_listen_address: "127.0.0.1:8090" alertmanager_listen_address: "127.0.0.1:9093" diff --git a/ansible/host_vars/monitoring.ooni.org/vault b/ansible/host_vars/monitoring.ooni.org/vault index c072ef476..cc3914652 100644 --- a/ansible/host_vars/monitoring.ooni.org/vault +++ b/ansible/host_vars/monitoring.ooni.org/vault @@ -1,26 +1,35 @@ $ANSIBLE_VAULT;1.1;AES256 -39323462306464623236623763633561303765353939336631373238643364326534396239303161 -6563643661653539623037616362343331333833663739310a663434656131366366613661616434 -61316134383261623761326634363633393362386237633666393134346266653432323363356239 -6634633238383262640a363363353434346566393938313235653933366130363364396534613732 -38643533626236333766323631626437366331663462383037376166353265636661316261386539 -63303934613465656261636632356566643233626136356639663135626436376337613731353730 -64613562643566663561623262383732663130393539636631626366326237353235396432663339 -66613038396338303663373763323338343536613365306331656464393061666166343633356539 -36353637656231626432366339613266343634383362636637373537663131646461626232636461 -31636139346539333032663632393030646435376330626365333834666531356433373234653664 -36383266376665346439663031323433613464366530666230393239393135356433363861306562 -30396663336330383536376239663430336239663862346461323865666336333163353136646264 -61626163363535363066393236613032376531346165623461356531616566653062356261353130 -31616535303433326432633433353861363563343437323736613733663439646564626166343531 -37306234363435336138616239306466663134613761386232303861323332303930386636343830 -36623531306431346639383266363234316461333065646631653532626431356166356333633137 -64636533333737383238633366653565303932366263353366306632623637373539303665303061 -61313164643634636262383230353931356163626466633436303165663631313833373063633165 -66343535623832353630326262326663633432613863633739643132333137653730336232383432 -61303432373533613361323138633361323633306234663865636136633733636439313561343566 -65363435393639393564643633346161653866383234393537396462633736343463623832353539 -32336236613138313538373738333565383531633033396231666262333338356237633337383033 -35613463396265373732633462646239366266613331313963383930356234336339386463303432 -39336538663037343233353166336330323562326132336539373031383265316438636439376336 -3764 +65633561393637336135303932663038376535343161636137396266323439633564616561643936 +3464383562346336613934383032333661353865333332610a666165633734363137356537306334 +38373534636162353635393934666135323163356163346632353538346434326662646561633336 +6265623261613463370a323762353338386562643336613834613862643636323261323131663030 +36626138393464326338613461383864653661343534303764653630393865643036356361613532 +31623965663231663938373936386236356265313734663839653961393338383765373737636261 +37626362323030353635363331363539316335313232663539316632316631383766643965323736 +65383138653237303632383638373332306262353465666261666366666438333832633939376439 +66393834343030393066656565306366376432376235613664386662343032383166383765306563 +33343338363434626662643562313239393638656539333164353263393064393031386135313762 +63353130376539666433323235376366326461623631333632386533653437383035613738333138 +66313265653562323930633737303937646239653965346632383238366230613736373133303862 +62643539613862656263303435656364353934626466616563383234316565383835393634346137 +65313064326165303665383137303565643530373934333237373233373432636362303530343332 +62316538383562323566653234373066373161363832313136393135663039616137656330343939 +35613333323538396135343333313536386662373061663639303163633861346235393434376562 +36333137346238306563333761303362623031613933373738613864396434323763383564343362 +39306665663066666463323665333636303634303239663461656132643932613861346331653435 +34366331323630623931363037303061636662373263653032623365663065663165383535373034 +33613339633466393063626437343838343536323861653836343138386332353438666161373866 +31383666316233653330643337656461653135393539336231333234666537666138306337343836 +64353938316138353438363338363831346332616337316663323433623338333661653035636663 +35306330373633313831373332663563303039396530663066356666326564343966356234363965 +32623662653839313966376234383232656466336638633761386139363339323032616331656336 +61633433656330636262343831613464326364316237366131333332353638646538323832653763 +36636337393138636137656636396633373131383537306230653432623261653065393839663039 +31383239663438366166643935633365663335393931653137613466613833333230656431363563 +37386463323237326439366463366533396539623136636138373336383838383232356233366264 +65613566343336616631346266373966313638383534316433626134376538303961646566373763 +30356161383066366361396164386362316532393361653838303337316236623533393535386533 +36383635363665366463316566373664383661346634633735396131343735323231613662393334 +63623164396230376439356536663730613739373436306332623433613339623564616361373561 +62396431643539326134663633636631346133396465363337396238376137383364373231346565 +3963343964383833383834333934303637643033633963626466 diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml index 7719bde2e..a3f469526 100644 --- a/ansible/roles/prometheus/defaults/main.yml +++ b/ansible/roles/prometheus/defaults/main.yml @@ -3,3 +3,4 @@ prometheus_ssl_dir: /etc/prometheus/ssl prometheus_exporter_cert: "/etc/prometheus/exporter_ca.cert" prometheus_basic_auth_user: "prometheus" prometheus_basic_auth_password: "{{ CHANGE_ME }}" +grafana_cloud_api_token: "{{ CHANGE_ME }}" diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index a6aeb27bd..b31b129e8 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -135,4 +135,9 @@ scrape_configs: - backend-hel.ooni.org:444 - ams-pg-test.ooni.org:444 +remote_write: + - url: https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push + basic_auth: + username: 1412863 + password: {{ grafana_cloud_api_token }} ... From 35e0492bc10d1e9dafca1dea03da3be0b3a55b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 4 Mar 2024 14:27:38 +0100 Subject: [PATCH 2/5] Fix auth key --- ansible/host_vars/monitoring.ooni.org/vault | 67 ++++++++++----------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/ansible/host_vars/monitoring.ooni.org/vault b/ansible/host_vars/monitoring.ooni.org/vault index cc3914652..d298787f6 100644 --- a/ansible/host_vars/monitoring.ooni.org/vault +++ b/ansible/host_vars/monitoring.ooni.org/vault @@ -1,35 +1,34 @@ $ANSIBLE_VAULT;1.1;AES256 -65633561393637336135303932663038376535343161636137396266323439633564616561643936 -3464383562346336613934383032333661353865333332610a666165633734363137356537306334 -38373534636162353635393934666135323163356163346632353538346434326662646561633336 -6265623261613463370a323762353338386562643336613834613862643636323261323131663030 -36626138393464326338613461383864653661343534303764653630393865643036356361613532 -31623965663231663938373936386236356265313734663839653961393338383765373737636261 -37626362323030353635363331363539316335313232663539316632316631383766643965323736 -65383138653237303632383638373332306262353465666261666366666438333832633939376439 -66393834343030393066656565306366376432376235613664386662343032383166383765306563 -33343338363434626662643562313239393638656539333164353263393064393031386135313762 -63353130376539666433323235376366326461623631333632386533653437383035613738333138 -66313265653562323930633737303937646239653965346632383238366230613736373133303862 -62643539613862656263303435656364353934626466616563383234316565383835393634346137 -65313064326165303665383137303565643530373934333237373233373432636362303530343332 -62316538383562323566653234373066373161363832313136393135663039616137656330343939 -35613333323538396135343333313536386662373061663639303163633861346235393434376562 -36333137346238306563333761303362623031613933373738613864396434323763383564343362 -39306665663066666463323665333636303634303239663461656132643932613861346331653435 -34366331323630623931363037303061636662373263653032623365663065663165383535373034 -33613339633466393063626437343838343536323861653836343138386332353438666161373866 -31383666316233653330643337656461653135393539336231333234666537666138306337343836 -64353938316138353438363338363831346332616337316663323433623338333661653035636663 -35306330373633313831373332663563303039396530663066356666326564343966356234363965 -32623662653839313966376234383232656466336638633761386139363339323032616331656336 -61633433656330636262343831613464326364316237366131333332353638646538323832653763 -36636337393138636137656636396633373131383537306230653432623261653065393839663039 -31383239663438366166643935633365663335393931653137613466613833333230656431363563 -37386463323237326439366463366533396539623136636138373336383838383232356233366264 -65613566343336616631346266373966313638383534316433626134376538303961646566373763 -30356161383066366361396164386362316532393361653838303337316236623533393535386533 -36383635363665366463316566373664383661346634633735396131343735323231613662393334 -63623164396230376439356536663730613739373436306332623433613339623564616361373561 -62396431643539326134663633636631346133396465363337396238376137383364373231346565 -3963343964383833383834333934303637643033633963626466 +30626430623165303966346666353063313865656563653237613262323662643331313561343032 +6666653434633562316466333465343863353138343265660a616530636436373064306432643261 +34383365353633396162313830353363653363656232366233623365643939346338633261616235 +6434636363356538320a623866386239663264636236343336643737636230323533393230343735 +62643066316165373562643537336634343437393264616433613165636538643662656164383338 +31376437303461323834336639383863616563656638626436303838373632303466653366323137 +66366238646163346337376531313462306436653032383463363430656632313938333262303964 +38656637643338643336656663386139356635343862653433363038323734393538333364303130 +31343631316166613564666331326435316134616238386535323930383036303436623130373764 +38663633383730663334346539326431663632336235313934373836386461353831623635623637 +36386131653134326539386464376264383233356438343738316538623238643930313261306563 +61386664353536666235303335613461363461306630623835616434323436653962303766623838 +61333539646638656439616162366339333465303737313537303566386432366236656638386135 +66653435323534643230396639376463393262386462623237616664386635396361366339313235 +37363732383538343166636137613438363965636135623963373236626366623062313061306539 +35616337356163666336666233363161366665383036633961323338363938303933333036333030 +30613231363965633532663134303339303563336432376233613962306365393931303237343266 +36303765313831376633323035306634323033393236366636646336366366383463313539656539 +36326361393837656337333431623638393363346337383266626361653134303263646461316661 +64336431393838353963663163353762646262363832383266393963613431653965656364613466 +66316465353461643335646638336666636562353664363031623635613965616466656433373432 +39356235326537353035386331323961363230316163353630396262663231663364386535333863 +63303561333537306339386636343733663535393362393966653762386431323933383031343261 +39613337333338333737393964653933656438336463343134313638623365363634656631333965 +65623136343237313236343463623832623631393239613338653531653837656239396638663433 +37356235343066363136333562313936396663373433383066396532313536386263323231623335 +64323563633134643231323838666434663433343331623363306633306632616335343866643337 +62643230363563613333386633613631303432616333653634306362366137383432626131366239 +32653962363763363531623938613138623231323161363434343734643736363763396233373739 +64303566663237643863353538323932313938393832326534663734623162356235306638333634 +65376465336438306337626663313539623063333232313037643061353462366562313335643762 +61396237626162306230376431393963643662353866663131333964363534373965356366306466 +36613361326237626264316431633465356661333635643862396466333234363439 From 38844d237c4b2137108400d0f121dd87a95839bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 5 Mar 2024 16:39:41 +0100 Subject: [PATCH 3/5] Bump scrape intervals to 60s --- .../roles/prometheus/templates/prometheus.yml.j2 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index b31b129e8..84dc09f98 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -1,7 +1,7 @@ --- global: - scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. - evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + scrape_interval: 60s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 60s # Evaluate rules every 15 seconds. The default is every 1 minute. scrape_timeout: 15s # the path is absolute as ansible `validates` temporary file in some temporary directory @@ -45,7 +45,7 @@ scrape_configs: {% endfor %} - job_name: 'node' - scrape_interval: 5s + scrape_interval: 60s scheme: https tls_config: ca_file: "{{ prometheus_exporter_cert }}" @@ -73,7 +73,7 @@ scrape_configs: # - targets: - job_name: 'raw-netdata' - scrape_interval: 5s + scrape_interval: 60s scheme: http metrics_path: "/api/v1/allmetrics" params: @@ -90,7 +90,7 @@ scrape_configs: # - 2.th.ooni.org:19999 - job_name: 'test-helpers' - scrape_interval: 5s + scrape_interval: 60s scheme: http metrics_path: "/metrics" static_configs: @@ -101,7 +101,7 @@ scrape_configs: - 3.th.ooni.org:9001 - job_name: 'ooni-api' - scrape_interval: 5s + scrape_interval: 60s scheme: https static_configs: - targets: [ 'api.ooni.io:443' ] @@ -118,7 +118,7 @@ scrape_configs: - ooni.netlify.com:443 - job_name: 'clickhouse' - scrape_interval: 5s + scrape_interval: 60s scheme: http metrics_path: "/metrics" static_configs: @@ -127,7 +127,7 @@ scrape_configs: # See ansible/roles/ooni-backend/tasks/main.yml for the scraping targets - job_name: 'haproxy' - scrape_interval: 5s + scrape_interval: 60s scheme: https metrics_path: "/__haproxy_prom_metrics" static_configs: From b4b1ba2164e03e3d4573c0397e15ace73f6f72e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 6 Mar 2024 12:55:42 +0100 Subject: [PATCH 4/5] Revert "Bump scrape intervals to 60s" This reverts commit 38844d237c4b2137108400d0f121dd87a95839bc. --- .../roles/prometheus/templates/prometheus.yml.j2 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index 84dc09f98..b31b129e8 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -1,7 +1,7 @@ --- global: - scrape_interval: 60s # Set the scrape interval to every 15 seconds. Default is every 1 minute. - evaluation_interval: 60s # Evaluate rules every 15 seconds. The default is every 1 minute. + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. scrape_timeout: 15s # the path is absolute as ansible `validates` temporary file in some temporary directory @@ -45,7 +45,7 @@ scrape_configs: {% endfor %} - job_name: 'node' - scrape_interval: 60s + scrape_interval: 5s scheme: https tls_config: ca_file: "{{ prometheus_exporter_cert }}" @@ -73,7 +73,7 @@ scrape_configs: # - targets: - job_name: 'raw-netdata' - scrape_interval: 60s + scrape_interval: 5s scheme: http metrics_path: "/api/v1/allmetrics" params: @@ -90,7 +90,7 @@ scrape_configs: # - 2.th.ooni.org:19999 - job_name: 'test-helpers' - scrape_interval: 60s + scrape_interval: 5s scheme: http metrics_path: "/metrics" static_configs: @@ -101,7 +101,7 @@ scrape_configs: - 3.th.ooni.org:9001 - job_name: 'ooni-api' - scrape_interval: 60s + scrape_interval: 5s scheme: https static_configs: - targets: [ 'api.ooni.io:443' ] @@ -118,7 +118,7 @@ scrape_configs: - ooni.netlify.com:443 - job_name: 'clickhouse' - scrape_interval: 60s + scrape_interval: 5s scheme: http metrics_path: "/metrics" static_configs: @@ -127,7 +127,7 @@ scrape_configs: # See ansible/roles/ooni-backend/tasks/main.yml for the scraping targets - job_name: 'haproxy' - scrape_interval: 60s + scrape_interval: 5s scheme: https metrics_path: "/__haproxy_prom_metrics" static_configs: From f5373b7507eb3d19ed73b0370b98e757a3b4fb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 6 Mar 2024 12:59:48 +0100 Subject: [PATCH 5/5] Disable grafana cloud while the billing is being sorted out --- ansible/roles/prometheus/templates/prometheus.yml.j2 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index b31b129e8..374ae714c 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -135,9 +135,9 @@ scrape_configs: - backend-hel.ooni.org:444 - ams-pg-test.ooni.org:444 -remote_write: - - url: https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push - basic_auth: - username: 1412863 - password: {{ grafana_cloud_api_token }} +#remote_write: +# - url: https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push +# basic_auth: +# username: 1412863 +# password: {{ grafana_cloud_api_token }} ...