Import gp_relaccess_stats into gpcontrib from Greenplum (open-gpdb#48) (#1954)

* Import gp_relaccess_stats into gpcontrib from Greenplum
* Feat: Adapt gp_relaccess_stats for Cloudberry
* Fix: Delete _PG_fini

---------
Co-authored-by: Maxim Smyatkin <smyatkinmaxim@gmail.com>

Author: Vlasdislav

.github/workflows/build-cloudberry.yml

@@ -279,6 +279,11 @@ jobs:
                 },
                 "enable_core_check":false
               },
+              {"test":"gpcontrib-gp-relaccess-stats",
+               "make_configs":["gpcontrib/gp_relaccess_stats:installcheck"],
+               "extension":"gp_relaccess_stats",
+               "shared_preload_libraries":"gp_relaccess_stats"
+              },
               {"test":"gpcontrib-gp-stats-collector",
                "make_configs":["gpcontrib/gp_stats_collector:installcheck"],
                "extension":"gp_stats_collector"
@@ -1576,6 +1581,20 @@ jobs:
                     exit 1
                 fi
                 ;;
+              gp_relaccess_stats)
+                if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \
+                  source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \
+                  gpconfig -c shared_preload_libraries -v 'gp_relaccess_stats' && \
+                  gpstop -ra && \
+                  echo 'CREATE EXTENSION IF NOT EXISTS gp_relaccess_stats; \
+                        SHOW shared_preload_libraries; \
+                        TABLE pg_extension;' | \
+                    psql postgres"
+                then
+                    echo "Error creating gp_relaccess_stats extension"
+                    exit 1
+                fi
+                ;;
               *)
                 echo "Unknown extension: ${{ matrix.extension }}"
                 exit 1

gpcontrib/Makefile

@@ -23,6 +23,7 @@ ifeq "$(enable_debug_extensions)" "yes"
                gp_exttable_fdw \
                gp_legacy_string_agg \
 			   gp_relsizes_stats \
+			   gp_relaccess_stats \
                gp_replica_check \
                gp_toolkit \
                pg_hint_plan \
@@ -33,6 +34,7 @@ else
                gp_internal_tools \
                gp_legacy_string_agg \
 			   gp_relsizes_stats \
+			   gp_relaccess_stats \
                gp_exttable_fdw \
                gp_toolkit \
                pg_hint_plan

gpcontrib/gp_relaccess_stats/.gitignore

@@ -0,0 +1,5 @@
+*.o
+*.so
+.vscode
+compile_commands.json
+results

gpcontrib/gp_relaccess_stats/Makefile

@@ -0,0 +1,19 @@
+MODULE_big      = gp_relaccess_stats
+OBJS            = ./src/gp_relaccess_stats.o
+EXTENSION       = gp_relaccess_stats
+EXTVERSION      = 1.0
+DATA            = $(wildcard sql/*--*.sql)
+REGRESS         = gp_relaccess_stats
+REGRESS_OPTS    = --inputdir=test/
+PGFILEDESC      = "gp_relaccess_stats - facility to track how and when tables, partitions or views were accessed"
+PG_CXXFLAGS     += $(COMMON_CPP_FLAGS)
+
+ifdef USE_PGXS
+	PG_CONFIG       = pg_config
+	PGXS            := $(shell $(PG_CONFIG) --pgxs)
+	include $(PGXS)
+else
+	top_builddir = ../..
+	include $(top_builddir)/src/Makefile.global
+	include $(top_srcdir)/contrib/contrib-global.mk
+endif

gpcontrib/gp_relaccess_stats/README.md

@@ -0,0 +1,94 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# gp_relaccess_stats: Table access monitoring tool for Greenplum
+
+## Features
+gp_relaccess_stats is an extension that records access statistics for Greenplum tables and views. Allowing users to see what objects were used, when and by whom. For example, this allows DBAs to find objects that are not used anymore or objects that are being misused.
+
+Features include:
+* support of both tables (regular, external or partitioned) and views
+* separate tracking of select, insert, update and delete queries
+* separate tracking of last read and write timestamps
+* tracking of the last user who accessed the object
+* per-database configuration
+* in-memory stats survive server restarts (but not crashes)
+
+### Supported versions and platforms
+For now it is being tested only for GP6 and Linux. Though, there are no apparent reasons why it should not be working on newer GP versions (or even PG with slight code modification) or other OSes.
+
+### Configuration
+As this extension does extensive usage of hooks and shared memory, you need to load gp_relaccess_stats.so on start-up:
+```
+gpconfig -c shared_preload_libraries -v '<old_shared_preload_libraries_contents,>gp_relaccess_stats' && gpstop -ra
+```
+gp_relaccess_stats configuration parameters:
+| **Parameter** | **Type**     | **Default**  | **Default**  |
+| ---------------- | --------------- | ------------ | ------------ |
+| `gp_relaccess_stats.enabled` | bool | false | Using `gp_relaccess_stats.enabled` you can enable/disable stats collection either globally or for each database separately. The second option is preferred.|
+| `gp_relaccess_stats.max_tables` | integer | 65536 | `gp_relaccess_stats.max_tables` is a hard limit on how many tables can be cached in shared memory. Feel free to make this number higher if necessary, as the overhead is only about 160 bytes per table. Note, that stats cache for a specific table is evicted from memory any time you execute `relaccess_stats_update()` or `relaccess_stats_dump()` and new tables can be recorded. If you call these functions often enough, there is no need for high gp_relaccess_stats.max_tables|
+| `gp_relaccess_stats.dump_on_overflow` | bool | false | This parameter configures what happens in case `gp_relaccess_stats.max_tables` was not enough. If set to `true`, `relaccess_stats_dump()` will be called implicitly and stats cache will be freed. Otherwice, you will get a WARNING saying that there is no room for new stats. Is this case, stats for some tables will be lost.|
+
+### Usage
+The first thing you need to do after `CREATE EXTENSION` and configuring - execute `SELECT relaccess_stats_init();` in a specific database. This function will fill `relaccess_stats` table with empty stats for each table and partition in this database. This is optional, but will come handy when you try to find tables that haven't been used recently, for example.
+
+Then, either manually or with a cron job start executing `select relaccess_stats_update()`. This function takes all stats cached in shared memory and all stats stored in pg_stat dir (e.g, dumps after restarts, or when `max_tables` was exceeded) and upserts them into `relaccess_stats` table.
+
+The `relaccess_stats` table itself looks like this:
+| **Column** | **Description**     |
+| ---------------- | --------------- |
+| relid | OID of the relation |
+| relname | Name of the relation at last access |
+| last_reader_id | OID of user who read the table last  |
+| last_writer_id | OID of user who wrote the table last  |
+| last_read | Timestamp of the most recent select |
+| last_write | Timestamp of the most recent insert/delete/update/truncate |
+| n_select_queries |  |
+| n_insert_queries |  |
+| n_update_queries |  |
+| n_delete_queries |  |
+| n_truncate_queries |  |
+
+**NOTE**: n_*_queries columns count the number of queries executed, not the number of rows read, inserted, deleted or updated.
+
+This table has a view associated with it: `relaccess_stats_root_tables_aggregated`. This view has exactly same columns, however it only shows partitioned tables. To be more specific, it shows aggregated stats for each partitioned table.
+For example, if we have 1 insert into `tbl1_prt_1` and 3 inserts into `tbl1_prt_2`, then `select * from relaccess_stats_root_tables_aggregated where relname = 'tbl1'` will show us only root table with n_insert_queries = 4. This view, however, has some limitations. See the next section for more detail.
+
+Another useful function is `relaccess_stats_dump()`, which simply moves cached stats from shared memory to temporary files in pg_stat directory. This function is cheaper than `relaccess_stats_update` but will evict stats cache if needed. Though, stats in temporary files can also get lost. Hence, it is recommended to stick with frequent `select relaccess_stats_update()` calls.
+
+To better understand when it's time to dump or update the stats one might check `select relaccess.relaccess_stats_fillfactor();`. It will show current usage of stats hash table in percents. For example if shared memory for our relaccess hash table is 70% full we will get relaccess_stats_fillfactor=70. It would be a good idea to dump or update when fillfactor is around 70%.
+
+### Limitations and gotchas
+There is a number of interesting edge-cases in this simple extension:
+* `relaccess_stats_root_tables_aggregated` shows info only about tables that exist **now**. We simply can`t get information about inheritance relationship for deleted tables.
+* Stats don't rollback on savepoint rollback. We will see n_select_queries incremented by 2 in the following case:
+```sql
+BEGIN;
+SELECT * FROM tbl;
+SAVEPOINT sp;
+SELECT * FROM tbl;
+ROLLBACK TO SAVEPOINT sp;
+COMMIT;
+```
+There is no technical reason for this limitation. It cat be fixed when there will be need for that.
+* Update stats often! Otherwise, data can be lost if any of it happens: 1) there was a crash, 2) `max_tables` exceeded w/o `dump_on_overflow`, 3) temporary pg_stat dir got cleaned.
+* no `truncate only` support. There is a TODO in code in case it is ever needed.
+* Updates and Deletes also increment n_select_queries. Every update and delete also read the table. That is, n_select_queries get incremented as well. If you need **only** selects, query like this `SELECT n_select_queries - (n_update_queries + n_delete_queries) ... FROM relaccess_stats ...;`. For this same reason last_read and last_reader_id change on update and delete queries.
+* view = view + tables. It looks like whenever you select from view, n_select_queries get incremented for both the view and tables it references.
+* obviously, we don't know any timestamps before we started tracking. So, the first timestamps are initialized with 0 (something around year 2000), which means those tables haven't been accessed since gp_relaccess_stats was enabled.

gpcontrib/gp_relaccess_stats/gp_relaccess_stats.control

@@ -0,0 +1,6 @@
+# gp_relaccess_stats extension
+comment = 'gp_relaccess_stats - facility to track how and when tables, partitions or views were accesseds'
+default_version = '1.1'
+module_pathname = '$libdir/gp_relaccess_stats'
+relocatable = true
+trusted = true

gpcontrib/gp_relaccess_stats/sql/gp_relaccess_stats--1.0--1.1.sql

@@ -0,0 +1,52 @@
+/* gp_relaccess_stats--1.0--1.1.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION gp_relaccess_stats UPDATE TO '1.1'" to load this file. \quit
+
+DROP VIEW relaccess.relaccess_stats_root_tables_aggregated;
+
+ALTER TABLE relaccess.relaccess_stats ALTER COLUMN n_select_queries TYPE int8;
+ALTER TABLE relaccess.relaccess_stats ALTER COLUMN n_insert_queries TYPE int8;
+ALTER TABLE relaccess.relaccess_stats ALTER COLUMN n_update_queries TYPE int8;
+ALTER TABLE relaccess.relaccess_stats ALTER COLUMN n_delete_queries TYPE int8;
+ALTER TABLE relaccess.relaccess_stats ALTER COLUMN n_truncate_queries TYPE int8;
+
+-- This utility view shows **ONLY** stats on **EXISTING** partitioned tables in aggregated form
+CREATE VIEW relaccess.relaccess_stats_root_tables_aggregated AS (
+    WITH RECURSIVE parents AS (
+        SELECT inhrelid AS child, inhparent AS parent FROM pg_inherits
+        UNION ALL
+        SELECT prev.child, next.inhparent AS parent FROM parents AS prev JOIN pg_inherits AS next ON prev.parent = next.inhrelid
+    ), part_to_root_mapping AS (
+        SELECT DISTINCT child AS partid, min(parent) OVER (partition BY child) AS rootid FROM parents
+    ), parts_including_roots AS (
+        SELECT rootid as partid, rootid FROM (SELECT DISTINCT rootid FROM part_to_root_mapping) AS p
+        UNION
+        SELECT * FROM part_to_root_mapping
+    ), with_root_id AS (
+        SELECT part_tbl.rootid, stats.* FROM relaccess.relaccess_stats stats JOIN parts_including_roots part_tbl ON (stats.relid = part_tbl.partid)
+    ), without_last_user AS (
+        SELECT rootid AS relid,
+            rootid::regclass::text AS relname,
+            max(last_read) AS last_read,
+            max(last_write) AS last_write,
+            sum(n_select_queries) AS n_select_queries,
+            sum(n_insert_queries) AS n_insert_queries,
+            sum(n_update_queries) AS n_update_queries,
+            sum(n_delete_queries) AS n_delete_queries,
+            sum(n_truncate_queries) AS n_truncate_queries
+        FROM with_root_id outer_tbl GROUP BY rootid
+    )
+    SELECT relid,
+        relname,
+        (SELECT last_reader_id FROM with_root_id w WHERE w.rootid = wo.relid AND wo.last_read = w.last_read LIMIT 1) AS last_reader_id,
+        (SELECT last_writer_id FROM with_root_id w WHERE w.rootid = wo.relid AND wo.last_write = w.last_write LIMIT 1) AS last_writer_id,
+        last_read,
+        last_write,
+        n_select_queries,
+        n_insert_queries,
+        n_update_queries,
+        n_delete_queries,
+        n_truncate_queries
+    FROM without_last_user wo
+);

gpcontrib/gp_relaccess_stats/sql/gp_relaccess_stats--1.1.sql

@@ -0,0 +1,143 @@
+/* gp_relaccess_stats--1.1.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION gp_relaccess_stats" to load this file. \quit
+
+CREATE SCHEMA IF NOT EXISTS relaccess;
+
+CREATE TABLE relaccess.relaccess_stats (
+    relid Oid,
+    relname Name,
+    last_reader_id Oid,
+    last_writer_id Oid,
+    last_read timestamptz,
+    last_write timestamptz,
+    n_select_queries int8,
+    n_insert_queries int8,
+    n_update_queries int8,
+    n_delete_queries int8,
+    n_truncate_queries int8
+) DISTRIBUTED BY (relid);
+
+CREATE FUNCTION relaccess.relaccess_stats_dump()
+RETURNS SETOF void
+AS 'MODULE_PATHNAME', 'relaccess_stats_dump'
+LANGUAGE C VOLATILE EXECUTE ON MASTER;
+
+CREATE FUNCTION relaccess.relaccess_stats_update()
+RETURNS SETOF void
+AS 'MODULE_PATHNAME', 'relaccess_stats_update'
+LANGUAGE C VOLATILE EXECUTE ON MASTER;
+
+CREATE FUNCTION relaccess.relaccess_stats_fillfactor()
+RETURNS SETOF INT2
+AS 'MODULE_PATHNAME', 'relaccess_stats_fillfactor'
+LANGUAGE C VOLATILE EXECUTE ON MASTER;
+
+CREATE FUNCTION relaccess.__get_db_stats_from_dump()
+RETURNS SETOF relaccess.relaccess_stats
+AS 'MODULE_PATHNAME', 'relaccess_stats_from_dump'
+LANGUAGE C VOLATILE EXECUTE ON MASTER;
+
+CREATE FUNCTION relaccess.__relaccess_upsert_from_dump_file() RETURNS VOID
+LANGUAGE plpgsql VOLATILE AS
+$func$
+BEGIN
+    EXECUTE 'DROP TABLE IF EXISTS relaccess_stats_tmp';
+    EXECUTE 'CREATE TEMP TABLE relaccess_stats_tmp (LIKE relaccess.relaccess_stats) distributed by (relid)';
+    EXECUTE 'DROP TABLE IF EXISTS relaccess_stats_tmp_aggregated';
+    EXECUTE 'CREATE TEMP TABLE relaccess_stats_tmp_aggregated (LIKE relaccess.relaccess_stats) distributed by (relid)';
+    EXECUTE 'INSERT INTO relaccess_stats_tmp SELECT * FROM relaccess.__get_db_stats_from_dump()';
+    EXECUTE 'WITH aggregated_wo_relname_and_user AS (
+        SELECT relid, max(last_read) AS last_read, max(last_write) AS last_write, sum(n_select_queries) AS n_select_queries,
+            sum(n_insert_queries) AS n_insert_queries, sum(n_update_queries) AS n_update_queries, sum(n_delete_queries) AS n_delete_queries, sum(n_truncate_queries) AS n_truncate_queries
+        FROM relaccess_stats_tmp GROUP BY relid
+    )
+    INSERT INTO relaccess_stats_tmp_aggregated
+    SELECT relid,
+        (SELECT relname FROM relaccess_stats_tmp w WHERE w.relid = wo.relid AND greatest(wo.last_read, wo.last_write) IN (w.last_read, w.last_write) LIMIT 1) AS relname,
+        (SELECT last_reader_id FROM relaccess_stats_tmp w WHERE w.relid = wo.relid AND wo.last_read = w.last_read LIMIT 1) AS last_reader_id,
+        (SELECT last_writer_id FROM relaccess_stats_tmp w WHERE w.relid = wo.relid AND wo.last_write = w.last_write LIMIT 1) AS last_writer_id,
+        last_read,
+        last_write,
+        n_select_queries,
+        n_insert_queries,
+        n_update_queries,
+        n_delete_queries,
+        n_truncate_queries FROM aggregated_wo_relname_and_user AS wo';
+    EXECUTE 'DROP TABLE IF EXISTS relaccess_stats_tmp';
+    EXECUTE 'INSERT INTO relaccess.relaccess_stats
+        SELECT relid, relname, last_reader_id, last_writer_id, last_read, last_write, 0, 0, 0, 0, 0
+        FROM relaccess_stats_tmp_aggregated stage
+        WHERE NOT EXISTS (
+            SELECT 1 FROM relaccess.relaccess_stats orig WHERE orig.relid = stage.relid)';
+    EXECUTE 'UPDATE relaccess.relaccess_stats orig SET
+        relname = stage.relname,
+        n_select_queries = orig.n_select_queries + stage.n_select_queries,
+        n_insert_queries = orig.n_insert_queries + stage.n_insert_queries,
+        n_update_queries = orig.n_update_queries + stage.n_update_queries,
+        n_delete_queries = orig.n_delete_queries + stage.n_delete_queries,
+        n_truncate_queries = orig.n_truncate_queries + stage.n_truncate_queries
+    FROM relaccess_stats_tmp_aggregated stage
+        WHERE orig.relid = stage.relid';
+    EXECUTE 'UPDATE relaccess.relaccess_stats orig SET
+        last_reader_id = stage.last_reader_id, last_read = stage.last_read
+    FROM relaccess_stats_tmp_aggregated stage
+        WHERE orig.relid = stage.relid AND orig.last_read < stage.last_read';
+    EXECUTE 'UPDATE relaccess.relaccess_stats orig SET
+        last_writer_id = stage.last_writer_id, last_write = stage.last_write
+    FROM relaccess_stats_tmp_aggregated stage
+        WHERE orig.relid = stage.relid AND orig.last_write < stage.last_write';
+    EXECUTE 'DROP TABLE IF EXISTS relaccess_stats_tmp_aggregated';
+END
+$func$;
+
+CREATE FUNCTION relaccess.relaccess_stats_init() RETURNS VOID AS
+$$
+    WITH relations AS (
+        SELECT oid as relid, relname, relowner FROM pg_catalog.pg_class WHERE relkind in ('r', 'v', 'm', 'f', 'p')
+    )
+    INSERT INTO relaccess.relaccess_stats
+        SELECT relid, relname, relowner, relowner, '2000-01-01 03:00:00', '2000-01-01 03:00:00', 0, 0, 0, 0, 0
+        FROM relations AS all_rels WHERE NOT EXISTS(SELECT 1 FROM relaccess.relaccess_stats orig WHERE orig.relid = all_rels.relid);
+$$ LANGUAGE SQL VOLATILE;
+
+-- This utility view shows **ONLY** stats on **EXISTING** partitioned tables in aggregated form
+CREATE VIEW relaccess.relaccess_stats_root_tables_aggregated AS (
+    WITH RECURSIVE parents AS (
+        SELECT inhrelid AS child, inhparent AS parent FROM pg_inherits
+        UNION ALL
+        SELECT prev.child, next.inhparent AS parent FROM parents AS prev JOIN pg_inherits AS next ON prev.parent = next.inhrelid
+    ), part_to_root_mapping AS (
+        SELECT DISTINCT child AS partid, min(parent) OVER (partition BY child) AS rootid FROM parents
+    ), parts_including_roots AS (
+        SELECT rootid as partid, rootid FROM (SELECT DISTINCT rootid FROM part_to_root_mapping) AS p
+        UNION
+        SELECT * FROM part_to_root_mapping
+    ), with_root_id AS (
+        SELECT part_tbl.rootid, stats.* FROM relaccess.relaccess_stats stats JOIN parts_including_roots part_tbl ON (stats.relid = part_tbl.partid)
+    ), without_last_user AS (
+        SELECT rootid AS relid,
+            rootid::regclass::text AS relname,
+            max(last_read) AS last_read,
+            max(last_write) AS last_write,
+            sum(n_select_queries) AS n_select_queries,
+            sum(n_insert_queries) AS n_insert_queries,
+            sum(n_update_queries) AS n_update_queries,
+            sum(n_delete_queries) AS n_delete_queries,
+            sum(n_truncate_queries) AS n_truncate_queries
+        FROM with_root_id outer_tbl GROUP BY rootid
+    )
+    SELECT relid,
+        relname,
+        (SELECT last_reader_id FROM with_root_id w WHERE w.rootid = wo.relid AND wo.last_read = w.last_read LIMIT 1) AS last_reader_id,
+        (SELECT last_writer_id FROM with_root_id w WHERE w.rootid = wo.relid AND wo.last_write = w.last_write LIMIT 1) AS last_writer_id,
+        last_read,
+        last_write,
+        n_select_queries,
+        n_insert_queries,
+        n_update_queries,
+        n_delete_queries,
+        n_truncate_queries
+    FROM without_last_user wo
+);