From 1428aa56ed243e69f71419b0d68876569adea5a1 Mon Sep 17 00:00:00 2001 From: hieule Date: Wed, 13 May 2026 12:33:44 +0100 Subject: [PATCH 1/3] Change RemoteWritePrometheus datasource setting change timeout value from number to string according to grafana docs Change cacheLevel and timeInterval Work to fix the timeout 504 issue on VM Power & Carbon dashboard --- .../roles/grafana/templates/cloud_datasource.yaml.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 b/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 index b4ef6189..a3f40d12 100644 --- a/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 +++ b/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 @@ -11,7 +11,9 @@ datasources: basicAuth: true basicAuthUser: {{ remote_write_prometheus_username }} jsonData: - timeout: 120 + timeout: "120" + timeInterval: "120" + cacheLevel: "High" secureJsonData: basicAuthPassword: {{ remote_write_prometheus_password }} From 5d4f936ec46efb46d968228d5617cd11bd4388e7 Mon Sep 17 00:00:00 2001 From: hieule Date: Wed, 13 May 2026 15:57:01 +0100 Subject: [PATCH 2/3] Add dataproxy timeout to grafana.ini Working to fix the issue that Grafana dashboard timeout when query large data --- grafana_monitoring/roles/grafana/templates/grafana.ini.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/grafana_monitoring/roles/grafana/templates/grafana.ini.j2 b/grafana_monitoring/roles/grafana/templates/grafana.ini.j2 index 8ca72737..7485f28c 100644 --- a/grafana_monitoring/roles/grafana/templates/grafana.ini.j2 +++ b/grafana_monitoring/roles/grafana/templates/grafana.ini.j2 @@ -32,3 +32,6 @@ log_level = info [security] admin_password="{{ grafana_admin_password }}" + +[dataproxy] +timeout = 1200 From a3b5ee19735f7c8720f4d9cb213d156d4a0f42d4 Mon Sep 17 00:00:00 2001 From: hieule Date: Wed, 13 May 2026 16:14:58 +0100 Subject: [PATCH 3/3] Increase Grafana log level from 'info' to 'debug' To find out why grafana log shows bad request 400 --- grafana_monitoring/roles/grafana/templates/grafana.ini.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana_monitoring/roles/grafana/templates/grafana.ini.j2 b/grafana_monitoring/roles/grafana/templates/grafana.ini.j2 index 7485f28c..0f96da93 100644 --- a/grafana_monitoring/roles/grafana/templates/grafana.ini.j2 +++ b/grafana_monitoring/roles/grafana/templates/grafana.ini.j2 @@ -28,7 +28,7 @@ role_attribute_path=contains(groups[*], 'stfc-cloud/admins') && 'Admin' || conta [log] -log_level = info +log_level = debug [security] admin_password="{{ grafana_admin_password }}"