gp_stats_collector: opt-in EXPLAIN (FORMAT JSON) plans, extension 1.2

gpcontrib/gp_stats_collector/Makefile

@@ -1,7 +1,7 @@
 MODULE_big = gp_stats_collector
 EXTENSION  = gp_stats_collector
 DATA       = $(wildcard *--*.sql)
-REGRESS    = gpsc_cursors gpsc_dist gpsc_select gpsc_utf8_trim gpsc_utility gpsc_guc_cache gpsc_uds
+REGRESS    = gpsc_cursors gpsc_dist gpsc_select gpsc_utf8_trim gpsc_utility gpsc_guc_cache gpsc_json_plan gpsc_uds
 
 PROTO_BASES = gpsc_plan gpsc_metrics gpsc_set_service
 PROTO_OBJS  = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES))

gpcontrib/gp_stats_collector/README.md

@@ -16,7 +16,12 @@ An extension for collecting greenplum query execution metrics and reporting them
 -   **What:** Triggers generation of the `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)` and captures it.
 -   **GUCs:** `gpsc.enable`, `gpsc.min_analyze_time`, `gpsc.enable_cdbstats`(ANALYZE), `gpsc.enable_analyze`(BUFFERS, TIMING, VERBOSE).
 
-#### 4. Other Metrics
+#### 4. `EXPLAIN (FORMAT JSON)` data
+-   **What:** In addition to the text plans, captures the structured `EXPLAIN (FORMAT JSON)`
+    payloads: `plan_json` at query start and `analyze_json` at query end.
+-   **GUC:** `gpsc.enable_json_plan` (default `off`). See [JSON plans](#json-plans) below.
+
+#### 5. Other Metrics
 -   **What:** Captures Instrument, Greenplum, System, Network, Interconnect, Spill metrics.
 -   **GUC:** `gpsc.enable`.
 
@@ -26,3 +31,40 @@ An extension for collecting greenplum query execution metrics and reporting them
 -   **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `gpsc.ignored_users_list`.
 -   **Trimming plans:** Query texts and execution plans are trimmed based on `gpsc.max_text_size` and `gpsc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB.
 -   **Analyze collection:** Analyze is sent if execution time exceeds `gpsc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `gpsc.enable_analyze` is true.
+-   **JSON plans:** `gpsc.enable_json_plan` (default `off`), see [JSON plans](#json-plans).
+
+### JSON plans
+
+`gpsc.enable_json_plan` (default `off`, `PGC_SUSET`) additionally collects the structured
+`EXPLAIN (FORMAT JSON)` plan. Consumers that render the plan graphically (for example PEV2)
+need it: the text plan does not expose Motion nodes in a machine-readable form.
+
+What is collected and when:
+
+- `plan_json` is built at the start of *every* query - a second `ExplainPrintPlan` pass.
+- `analyze_json` is built at the end of the query under the same rules as `analyze_text`
+  (`gpsc.enable_analyze` plus the `gpsc.min_analyze_time` threshold), so only long queries
+  pay for it.
+- The text plans are unaffected: they are always collected, and `plan_id` is still the hash
+  of the normalized *text* plan.
+
+Why not derive it from the text plan: the text plan is a rendering, not a format. Its tree
+structure is encoded in indentation, and Greenplum adds annotations of its own
+(`Gather Motion 2:1 (slice1; segments: 2)`, `Rows out: Avg ... x N workers`, ORCA cost lines,
+`Executor memory`). Tools that parse text plans (PEV2 `fromText`) silently lose the `Motion`
+nodes and their subtrees, and the per-segment ANALYZE figures are prose. A text parser would
+be reverse engineering of `explain.c` output for one Greenplum version, while
+`EXPLAIN (FORMAT JSON)` is produced by the same tree walk and stays stable across versions.
+
+Cost and limits:
+
+- JSON is 3-5 times larger than the text plan, so the UDS message grows accordingly.
+- `gpsc.max_plan_size` matters more here: a JSON payload that does not fit the limit is
+  dropped whole rather than truncated (a truncated JSON is unparseable), while the text plan
+  is trimmed as usual.
+
+Table logging (`gpsc.logging_mode = 'TBL'`): the payloads land in the `plan_json` /
+`analyze_json` columns of `gpsc.log`. The columns were added in extension version 1.2, so an
+installation created with 1.1 needs `ALTER EXTENSION gp_stats_collector UPDATE` to get them.
+Until that is done the extension detects the column-count mismatch, emits a one-time
+`WARNING` per backend and skips table logging instead of writing misaligned rows.

gpcontrib/gp_stats_collector/expected/gpsc_guc_cache.out

@@ -22,6 +22,15 @@ SET gpsc.ignored_users_list TO '';
 SET gpsc.enable TO TRUE;
 SET gpsc.enable_utility TO TRUE;
 SET gpsc.logging_mode TO 'TBL';
+-- SET of the JSON GUC is a plan-less utility statement, still logged as SUBMIT + DONE.
+SET gpsc.enable_json_plan TO TRUE;
+SELECT * FROM print_last_query('SET gpsc.enable_json_plan TO TRUE;');
+    query_status     
+---------------------
+ QUERY_STATUS_SUBMIT
+ QUERY_STATUS_DONE
+(2 rows)
+
 -- SET below disables utility logging and DONE must still be logged.
 SET gpsc.enable_utility TO FALSE;
 SELECT * FROM print_last_query('SET gpsc.enable_utility TO FALSE;');
@@ -53,5 +62,6 @@ DROP FUNCTION print_last_query(text);
 DROP EXTENSION gp_stats_collector;
 RESET gpsc.enable;
 RESET gpsc.enable_utility;
+RESET gpsc.enable_json_plan;
 RESET gpsc.ignored_users_list;
 RESET gpsc.logging_mode;

gpcontrib/gp_stats_collector/expected/gpsc_json_plan.out

@@ -0,0 +1,184 @@
+CREATE EXTENSION gp_stats_collector;
+SET gpsc.ignored_users_list TO '';
+SET gpsc.enable TO TRUE;
+SET gpsc.enable_utility TO FALSE;
+SET gpsc.enable_analyze TO TRUE;
+-- Collect analyze for every query, regardless of its duration.
+SET gpsc.min_analyze_time TO 0;
+-- Test 1: gpsc.enable_json_plan off (set explicitly) => text plans only.
+SET gpsc.enable_json_plan TO FALSE;
+SET gpsc.logging_mode TO 'TBL';
+SELECT /*json_off*/ COUNT(*) FROM generate_series(1,10);
+ count 
+-------
+    10
+(1 row)
+
+RESET gpsc.logging_mode;
+SELECT plan_text IS NOT NULL AS has_plan_text,
+       plan_json IS NULL AS no_plan_json,
+       analyze_text IS NOT NULL AS has_analyze_text,
+       analyze_json IS NULL AS no_analyze_json
+FROM gpsc.log
+WHERE segid = -1 AND query_text LIKE '%json_off%' AND query_status = 'QUERY_STATUS_DONE';
+ has_plan_text | no_plan_json | has_analyze_text | no_analyze_json 
+---------------+--------------+------------------+-----------------
+ t             | t            | t                | t
+(1 row)
+
+-- Test 2: GUC on => valid JSON array with a "Plan" object for plan and analyze.
+SET gpsc.enable_json_plan TO TRUE;
+SET gpsc.logging_mode TO 'TBL';
+SELECT /*json_on*/ COUNT(*) FROM generate_series(1,10);
+ count 
+-------
+    10
+(1 row)
+
+RESET gpsc.logging_mode;
+SELECT json_typeof(plan_json::json) AS plan_json_type,
+       (plan_json::json->0->'Plan') IS NOT NULL AS has_plan_node,
+       json_typeof(analyze_json::json) AS analyze_json_type,
+       (analyze_json::json->0->'Plan') IS NOT NULL AS has_analyze_node,
+       analyze_json <> plan_json AS analyze_differs
+FROM gpsc.log
+WHERE segid = -1 AND query_text LIKE '%json_on%' AND query_status = 'QUERY_STATUS_DONE';
+ plan_json_type | has_plan_node | analyze_json_type | has_analyze_node | analyze_differs 
+----------------+---------------+-------------------+------------------+-----------------
+ array          | t             | array             | t                | t
+(1 row)
+
+-- The text plan (and thus plan_id) is the same with the GUC on and off.
+SELECT COUNT(DISTINCT plan_text) = 1 AS same_plan_text,
+       COUNT(DISTINCT plan_id) = 1 AS same_plan_id
+FROM gpsc.log
+WHERE segid = -1 AND query_text LIKE '%json_o%' AND query_status = 'QUERY_STATUS_DONE';
+ same_plan_text | same_plan_id 
+----------------+--------------
+ t              | t
+(1 row)
+
+-- Test 3: oversized JSON is dropped whole, the text plan is trimmed as usual (NULL at limit 0).
+SET gpsc.max_plan_size TO 0;
+SET gpsc.logging_mode TO 'TBL';
+SELECT /*json_big*/ COUNT(*) FROM generate_series(1,10);
+ count 
+-------
+    10
+(1 row)
+
+RESET gpsc.logging_mode;
+RESET gpsc.max_plan_size;
+SELECT coalesce(octet_length(plan_text), 0) AS plan_text_len,
+       plan_json IS NULL AS no_plan_json,
+       analyze_json IS NULL AS no_analyze_json
+FROM gpsc.log
+WHERE segid = -1 AND query_text LIKE '%json_big%' AND query_status = 'QUERY_STATUS_DONE';
+ plan_text_len | no_plan_json | no_analyze_json 
+---------------+--------------+-----------------
+             0 | t            | t
+(1 row)
+
+-- Test 4: analyze_json follows gpsc.enable_analyze.
+SET gpsc.enable_analyze TO FALSE;
+SET gpsc.logging_mode TO 'TBL';
+SELECT /*json_noanalyze*/ COUNT(*) FROM generate_series(1,10);
+ count 
+-------
+    10
+(1 row)
+
+RESET gpsc.logging_mode;
+SELECT plan_json IS NOT NULL AS has_plan_json,
+       analyze_json IS NULL AS no_analyze_json
+FROM gpsc.log
+WHERE segid = -1 AND query_text LIKE '%json_noanalyze%' AND query_status = 'QUERY_STATUS_DONE';
+ has_plan_json | no_analyze_json 
+---------------+-----------------
+ t             | t
+(1 row)
+
+-- Test 5: explain_memory_verbosity adds text-only "Memory:" lines; analyze_json stays valid JSON.
+SET explain_memory_verbosity TO detail;
+SET gpsc.enable_analyze TO TRUE;
+SET gpsc.logging_mode TO 'TBL';
+SELECT /*json_memverb*/ COUNT(*) FROM generate_series(1,10);
+ count 
+-------
+    10
+(1 row)
+
+RESET gpsc.logging_mode;
+RESET explain_memory_verbosity;
+SELECT json_typeof(analyze_json::json) AS analyze_json_type,
+       (analyze_json::json->0->'Plan') IS NOT NULL AS has_analyze_node,
+       analyze_json NOT LIKE '%Memory:  %' AS no_memory_text
+FROM gpsc.log
+WHERE segid = -1 AND query_text LIKE '%json_memverb%' AND query_status = 'QUERY_STATUS_DONE';
+ analyze_json_type | has_analyze_node | no_memory_text 
+-------------------+------------------+----------------
+ array             | t                | t
+(1 row)
+
+-- Test 6: a distributed join has a Motion node and nested plans in plan_json and analyze_json.
+CREATE TABLE gpsc_json_dist (id int, v int) DISTRIBUTED BY (id);
+INSERT INTO gpsc_json_dist SELECT i, i FROM generate_series(1,5) i;
+SET gpsc.logging_mode TO 'TBL';
+SELECT /*json_motion*/ COUNT(*) FROM gpsc_json_dist a JOIN gpsc_json_dist b ON a.id = b.id;
+ count 
+-------
+     5
+(1 row)
+
+RESET gpsc.logging_mode;
+SELECT plan_json LIKE '%Motion%' AS plan_has_motion,
+       (plan_json::json->0->'Plan'->'Plans') IS NOT NULL AS plan_has_subplans,
+       analyze_json LIKE '%Motion%' AS analyze_has_motion,
+       (analyze_json::json->0->'Plan'->'Plans') IS NOT NULL AS analyze_has_subplans
+FROM gpsc.log
+WHERE segid = -1 AND query_text LIKE '%json_motion%' AND query_status = 'QUERY_STATUS_DONE';
+ plan_has_motion | plan_has_subplans | analyze_has_motion | analyze_has_subplans 
+-----------------+-------------------+--------------------+----------------------
+ t               | t                 | t                  | t
+(1 row)
+
+-- Test 7: every GUC that extends EXPLAIN output keeps plan_json and analyze_json valid.
+SET explain_memory_verbosity TO debug;
+SET gp_enable_explain_allstat TO on;
+SET gp_enable_explain_rows_out TO on;
+SET gp_enable_explain_node_summary TO on;
+SET gpsc.logging_mode TO 'TBL';
+SELECT /*json_gucs*/ COUNT(*) FROM gpsc_json_dist a JOIN gpsc_json_dist b ON a.id = b.id;
+ count 
+-------
+     5
+(1 row)
+
+RESET gpsc.logging_mode;
+RESET gp_enable_explain_node_summary;
+RESET gp_enable_explain_rows_out;
+RESET gp_enable_explain_allstat;
+RESET explain_memory_verbosity;
+SELECT json_typeof(plan_json::json) AS plan_json_type,
+       json_typeof(analyze_json::json) AS analyze_json_type
+FROM gpsc.log
+WHERE segid = -1 AND query_text LIKE '%json_gucs%' AND query_status = 'QUERY_STATUS_DONE';
+ plan_json_type | analyze_json_type 
+----------------+-------------------
+ array          | array
+(1 row)
+
+DROP TABLE gpsc_json_dist;
+SELECT gpsc.truncate_log() IS NOT NULL AS t;
+ t 
+---
+ t
+(1 row)
+
+DROP EXTENSION gp_stats_collector;
+RESET gpsc.enable_json_plan;
+RESET gpsc.enable_analyze;
+RESET gpsc.min_analyze_time;
+RESET gpsc.enable;
+RESET gpsc.enable_utility;
+RESET gpsc.ignored_users_list;

gpcontrib/gp_stats_collector/gp_stats_collector--1.1--1.2.sql

@@ -0,0 +1,16 @@
+/* gp_stats_collector--1.1--1.2.sql */
+
+-- complain if script is sourced in psql, rather than via ALTER EXTENSION
+\echo Use "ALTER EXTENSION gp_stats_collector UPDATE TO '1.2'" to load this file. \quit
+
+-- EXPLAIN (FORMAT JSON) payloads; appended at the end, order must match LogSchema.h.
+ALTER TABLE gpsc.__log ADD COLUMN plan_json text;
+ALTER TABLE gpsc.__log ADD COLUMN analyze_json text;
+
+-- Recreate the view so that it picks up the new columns.
+DROP VIEW gpsc.log;
+CREATE VIEW gpsc.log AS
+  SELECT * FROM gpsc.__log -- master
+  UNION ALL
+  SELECT * FROM gp_dist_random('gpsc.__log') -- segments
+  ORDER BY tmid, ssid, ccnt;

gpcontrib/gp_stats_collector/gp_stats_collector--1.2.sql

@@ -0,0 +1,110 @@
+/* gp_stats_collector--1.2.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION gp_stats_collector" to load this file. \quit
+
+CREATE SCHEMA gpsc;
+
+CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master()
+RETURNS void
+AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset'
+LANGUAGE C EXECUTE ON MASTER;
+
+CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments()
+RETURNS void
+AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset'
+LANGUAGE C EXECUTE ON ALL SEGMENTS;
+
+CREATE FUNCTION gpsc.stat_messages_reset()
+RETURNS void
+AS
+$$
+  SELECT gpsc.__stat_messages_reset_f_on_master();
+  SELECT gpsc.__stat_messages_reset_f_on_segments();
+$$
+LANGUAGE SQL EXECUTE ON MASTER;
+
+CREATE FUNCTION gpsc.__stat_messages_f_on_master()
+RETURNS SETOF record
+AS 'MODULE_PATHNAME', 'gpsc_stat_messages'
+LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER;
+
+CREATE FUNCTION gpsc.__stat_messages_f_on_segments()
+RETURNS SETOF record
+AS 'MODULE_PATHNAME', 'gpsc_stat_messages'
+LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS;
+
+CREATE VIEW gpsc.stat_messages AS
+  SELECT C.*
+	FROM gpsc.__stat_messages_f_on_master() as C (
+    segid int,
+    total_messages bigint,
+    send_failures bigint,
+    connection_failures bigint,
+    other_errors bigint,
+    max_message_size int
+	)
+  UNION ALL
+  SELECT C.*
+	FROM gpsc.__stat_messages_f_on_segments() as C (
+    segid int,
+    total_messages bigint,
+    send_failures bigint,
+    connection_failures bigint,
+    other_errors bigint,
+    max_message_size int
+	)
+ORDER BY segid;
+
+CREATE FUNCTION gpsc.__init_log_on_master()
+RETURNS void
+AS 'MODULE_PATHNAME', 'gpsc_init_log'
+LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER;
+
+CREATE FUNCTION gpsc.__init_log_on_segments()
+RETURNS void
+AS 'MODULE_PATHNAME', 'gpsc_init_log'
+LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS;
+
+-- Creates log table inside gpsc schema.
+SELECT gpsc.__init_log_on_master();
+SELECT gpsc.__init_log_on_segments();
+
+CREATE VIEW gpsc.log AS
+  SELECT * FROM gpsc.__log -- master
+  UNION ALL
+  SELECT * FROM gp_dist_random('gpsc.__log') -- segments
+ORDER BY tmid, ssid, ccnt;
+
+CREATE FUNCTION gpsc.__truncate_log_on_master()
+RETURNS void
+AS 'MODULE_PATHNAME', 'gpsc_truncate_log'
+LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER;
+
+CREATE FUNCTION gpsc.__truncate_log_on_segments()
+RETURNS void
+AS 'MODULE_PATHNAME', 'gpsc_truncate_log'
+LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS;
+
+CREATE FUNCTION gpsc.truncate_log()
+RETURNS void AS $$
+BEGIN
+    PERFORM gpsc.__truncate_log_on_master();
+    PERFORM gpsc.__truncate_log_on_segments();
+END;
+$$ LANGUAGE plpgsql VOLATILE;
+
+CREATE FUNCTION gpsc.__test_uds_start_server(path text)
+RETURNS void
+AS 'MODULE_PATHNAME', 'gpsc_test_uds_start_server'
+LANGUAGE C STRICT EXECUTE ON MASTER;
+
+CREATE FUNCTION gpsc.__test_uds_receive(timeout_ms int DEFAULT 2000)
+RETURNS bigint
+AS 'MODULE_PATHNAME', 'gpsc_test_uds_receive'
+LANGUAGE C STRICT EXECUTE ON MASTER;
+
+CREATE FUNCTION gpsc.__test_uds_stop_server()
+RETURNS void
+AS 'MODULE_PATHNAME', 'gpsc_test_uds_stop_server'
+LANGUAGE C EXECUTE ON MASTER;