WIP: Statistics visualization with Grafana and InfluxDB

build.gradle

@@ -229,5 +229,7 @@ project(":core") {
         api 'de.ruedigermoeller:fst:2.56'
 
         api "org.joml:joml:1.10.5"
+
+        implementation "com.influxdb:influxdb-client-java:6.6.0"
     }
 }

core/src/com/protoevo/core/Simulation.java

@@ -9,6 +9,7 @@
 import com.protoevo.settings.SimulationSettings;
 import com.protoevo.utils.EnvironmentImageRenderer;
 import com.protoevo.utils.FileIO;
+import com.protoevo.utils.InfluxWriter;
 import com.protoevo.utils.TimedEventsManager;
 import com.protoevo.utils.Utils;
 
@@ -44,6 +45,7 @@ public class Simulation implements Runnable
 	private String name;
 	private List<String> statsNames;
 	private final REPL repl = new REPL(this);
+	private InfluxWriter influxWriter = new InfluxWriter();
 
 	public Simulation() {
 		this(Environment.settings.simulationSeed.get());
@@ -389,20 +391,22 @@ public void createAutoSave() {
 
 	public void makeStatisticsSnapshot() {
 		Statistics stats = new Statistics(environment.getStats());
+
 		stats.putAll(environment.getDebugStats());
 		stats.putAll(environment.getPhysicsDebugStats());
 		stats.putAll(environment.getProtozoaSummaryStats(true, false, true));
 
 		String timeStamp = Utils.getTimeStampString();
+		influxWriter.write(name, stats);
 
-		FileIO.writeJson(stats, getSaveFolder() + "/stats/summaries/" + timeStamp);
+		//FileIO.writeJson(stats, getSaveFolder() + "/stats/summaries/" + timeStamp);
 
 		if (Environment.settings.misc.writeGenomes.get()) {
 			List<NetworkGenome> protozoaGenomes = environment.getCells().stream()
 					.filter(cell -> cell instanceof Protozoan)
 					.map(cell -> ((Protozoan) cell).getGeneExpressionFunction().getGRNGenome())
 					.collect(Collectors.toList());
-			FileIO.writeJson(protozoaGenomes, getSaveFolder() + "/stats/protozoa-genomes/" + timeStamp);
+			//FileIO.writeJson(protozoaGenomes, getSaveFolder() + "/stats/protozoa-genomes/" + timeStamp);
 		}
 
 //		PythonRunner.runPython("pyprotoevo.create_plots", "--quiet --simulation " + name);

core/src/com/protoevo/settings/MiscSettings.java

@@ -22,7 +22,7 @@ public class MiscSettings extends Settings {
     public final Settings.Parameter<Float> statisticsSnapshotTime = new Settings.Parameter<>(
             "Save Statistics Time",
             "Amount of in-simulation time between making a new snapshot of the summary statistics.",
-            20.0f, Statistics.ComplexUnit.TIME);
+            2.0f, Statistics.ComplexUnit.TIME);
     public final Settings.Parameter<Boolean> writeGenomes = new Settings.Parameter<>(
             "Write Genomes",
             "Whether or not to write genome information (warning: generates lots of data).",

core/src/com/protoevo/test/InfluxDBTest.java

@@ -0,0 +1,33 @@
+package com.protoevo.test;
+
+import java.time.Instant;
+import java.util.List;
+
+import com.influxdb.annotations.Column;
+import com.influxdb.annotations.Measurement;
+import com.influxdb.client.InfluxDBClient;
+import com.influxdb.client.InfluxDBClientFactory;
+import com.influxdb.client.WriteApi;
+import com.influxdb.client.WriteApiBlocking;
+import com.influxdb.client.domain.WritePrecision;
+import com.influxdb.client.write.Point;
+import com.influxdb.query.FluxTable;
+
+public class InfluxDBTest {
+    public static void main(String[] args) {
+        String token = "my-super-secret-auth-token";
+        String bucket = "my-bucket";
+        String org = "protoevo";
+
+        InfluxDBClient client = InfluxDBClientFactory.create("http://localhost:8086", token.toCharArray());
+
+        Point point = Point
+                .measurement("mem")
+                .addTag("host", "host1")
+                .addField("used_percent", 23.43234543)
+                .time(Instant.now(), WritePrecision.NS);
+
+        WriteApiBlocking writeApi = client.getWriteApiBlocking();
+        writeApi.writePoint(bucket, org, point);
+    }
+}

core/src/com/protoevo/utils/InfluxWriter.java

@@ -0,0 +1,43 @@
+package com.protoevo.utils;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import com.influxdb.annotations.Column;
+import com.influxdb.annotations.Measurement;
+import com.influxdb.client.InfluxDBClient;
+import com.influxdb.client.InfluxDBClientFactory;
+import com.influxdb.client.WriteApi;
+import com.influxdb.client.WriteApiBlocking;
+import com.influxdb.client.domain.WritePrecision;
+import com.influxdb.client.write.Point;
+import com.influxdb.query.FluxTable;
+import com.protoevo.core.Statistics;
+import com.protoevo.core.Statistics.Stat;
+
+public class InfluxWriter {
+    private InfluxDBClient client;
+    private String token = "my-super-secret-auth-token";
+    private String bucket = "my-bucket";
+    private String org = "protoevo";
+
+    public InfluxWriter() {
+        client = InfluxDBClientFactory.create("http://localhost:8086", token.toCharArray());
+    }
+
+    public void write(String simulationName, Statistics stats) {
+        Point point = Point
+                .measurement("environment")
+                .addTag("simulation", simulationName);
+        for (Map.Entry<String, Stat> entry : stats.getStatsMap().entrySet()) {
+            Stat val = entry.getValue();
+            point.addField(entry.getKey(), val.getDouble());
+        }
+
+        WriteApiBlocking writeApi = client.getWriteApiBlocking();
+        writeApi.writePoint(bucket, org, point);
+    }
+}

stats_server/README.md

@@ -0,0 +1,63 @@
+# Stats server
+Statistics server consists of two containers, influxdb and grafana. They can be started with docker-compose. The protoevo Java application is configured to send data to the influxdb container.
+
+## Setup
+Startup the containers:
+(In detached mode)
+`docker-compose up -d`
+
+If you wish to look at the logs, you can either omit the `-d` flag or use `docker-compose logs -f`.
+
+### InfluxDB
+InfluxDB is available in the url `http://localhost:8086/`, the username is `influx_admin` and the password is `protoevo_influx` as defined in the `docker-compose.yml` file.
+
+### Grafana
+Grafana is available in the url `http://localhost:3000/`, the username is `admin` and the password is `stats_grafana` as defined in the `docker-compose.yml` file.
+
+The InfluxDB data source needs to be added manually to Grafana. From the toggle menu, choose connections -> data sources. Then add a new data source. Choose InfluxDB as the type and fill in the following fields:
+- Query Language: `Flux`
+- URL: `http://influxdb:8086` in HTTP section
+- Basic auth: uncheck in Auth section
+- Organization: `protoevo` in influxDB Details section
+- Token: `my-super-secret-auth-token` in influxDB Details section
+
+Everything else can be left as empty and all auth check boxes can be left unchecked.
+
+![data source settings](img/data_source_settings.jpg)
+
+Then click save and test. If everything is working, you should see a green notification saying "Data source is working".
+
+Now you can start protoevo and it should start sending data to the database. You can create dashboards and panels to visualize the data.
+
+To verify that data is flowing correctly all the way to Grafana, you can go to the explore tab and run a query. For example, you can run the following query:
+```flux
+from(bucket: "my-bucket")
+  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
+  |> filter(fn: (r) => r["_measurement"] == "environment")
+  |> filter(fn: (r) => r["_field"] == "Age Max")
+  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)
+  |> yield(name: "mean")
+```
+You should see a graph showing the maximum age of the protozoa over time.
+
+### Importing existing dashboards
+There's an example dashboard, `main_dashboard.json`, in the `example_queries` folder. You can import it to Grafana, but first you'll need to change the data source id to match the id of the InfluxDB data source you created earlier. You can find the id by going to the data sources page and clicking on the data source. The id is in the url. For example for a url like `http://localhost:3000/connections/datasources/edit/c7c7acfc-6644-4276-b55e-a0065f069bab` the id is `c7c7acfc-6644-4276-b55e-a0065f069bab`.
+
+All instances of `REPLACE_ME_WITH_CORRECT_ID` in the main_dashboard.json file should be replaced with the id of the InfluxDB data source.
+
+Then you can import the dashboard by going to the dashboard page and clicking on the plus icon on the left. Then choose import and upload the main_dashboard.json file.
+
+![imported dashboard](img/imported_dashboard.jpg)
+
+### Creating new queries
+Understanding `flux` is not necessary to create new queries. If you log in to the InfluxDB UI, you can go to the Data Explorer tab and use Query Builder to create queries. Once you're happy with the query, you can switch to the script editor to see the `flux` code. You can copy the code to Grafana and modify it there.
+
+For example you can create a query that shows Age Max, Age Min and Age Mean with the query builder like this:
+![query builder](img/query_builder.jpg)
+
+And then switch to the script editor to see the `flux` code:
+![script editor](img/script_editor.jpg)
+
+## Shutdown and data deletion
+Shutdown the containers with `docker-compose down`. This will keep the data in the volumes. If you wish to delete the data as well, use
+`docker-compose down -v`

stats_server/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '2'
+services:
+  influxdb:
+    image: influxdb:latest
+    ports:
+      - '8086:8086'
+    volumes:
+      - influxdb-storage:/var/lib/influxdb
+    environment:
+      - INFLUXDB_DB=db0
+      - DOCKER_INFLUXDB_INIT_MODE=setup
+      - DOCKER_INFLUXDB_INIT_USERNAME=influx_admin
+      - DOCKER_INFLUXDB_INIT_PASSWORD=protoevo_influx
+      - DOCKER_INFLUXDB_INIT_ORG=protoevo
+      - DOCKER_INFLUXDB_INIT_BUCKET=my-bucket
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
+
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - '3000:3000'
+    volumes:
+      - grafana-storage:/var/lib/grafana
+      - ./grafana-provisioning/:/etc/grafana/provisioning
+    depends_on:
+      - influxdb
+    environment:
+      - GF_SECURITY_ADMIN_USER=protoevo
+      - GF_SECURITY_ADMIN_PASSWORD=stats_grafana
+volumes:
+  influxdb-storage:
+  grafana-storage: